Uname:Linux antigravity-cli 6.8.0-31-generic #31-Ubuntu SMP PREEMPT_DYNAMIC Sat Apr 20 00:40:06 UTC 2024 x86_64

Base Dir : /var/www/moonbloom

User : wp-moonbloom


403WebShell
403Webshell
Server IP : 85.155.190.233  /  Your IP : 216.73.216.103
Web Server : nginx/1.24.0
System : Linux antigravity-cli 6.8.0-31-generic #31-Ubuntu SMP PREEMPT_DYNAMIC Sat Apr 20 00:40:06 UTC 2024 x86_64
User : wp-moonbloom ( 1001)
PHP Version : 8.3.6
Disable Function : NONE
MySQL : OFF  |  cURL : ON  |  WGET : ON  |  Perl : ON  |  Python : OFF  |  Sudo : ON  |  Pkexec : OFF
Directory :  /opt/Gemini_Swarm_Core/scripts/

Upload File :
current_dir [ Writeable ] document_root [ Writeable ]

 

Command :


[ Back ]     

Current File : /opt/Gemini_Swarm_Core/scripts/trend_scout.py
import asyncio
import os
import sys
import logging
from datetime import datetime, timezone, timedelta
from pathlib import Path

from aiogram import Bot

logger = logging.getLogger(__name__)

class TrendScoutManager:
    def __init__(self, bot: Bot, db_path: Path, allowed_users: list[str]):
        """Initialize bot instance, database path, and target user IDs."""
        self.bot = bot
        self.db_path = db_path
        self.allowed_users = allowed_users

    async def get_expected_last_run(self, now: datetime) -> datetime:
        """Returns the datetime of the last expected 01:00 UTC run (Monday or Thursday) relative to the current time."""
        expected_today = datetime(now.year, now.month, now.day, 1, 0, 0, tzinfo=timezone.utc)
        weekday = now.weekday()  # 0=Monday, 1=Tuesday, 2=Wednesday, 3=Thursday, 4=Friday, 5=Saturday, 6=Sunday
        
        if weekday == 0:  # Monday
            if now >= expected_today:
                return expected_today
            else:
                return expected_today - timedelta(days=4)  # Previous Thursday
        elif weekday == 1:  # Tuesday
            return expected_today - timedelta(days=1)  # Previous Monday
        elif weekday == 2:  # Wednesday
            return expected_today - timedelta(days=2)  # Previous Monday
        elif weekday == 3:  # Thursday
            if now >= expected_today:
                return expected_today
            else:
                return expected_today - timedelta(days=3)  # Previous Monday
        elif weekday == 4:  # Friday
            return expected_today - timedelta(days=1)  # Previous Thursday
        elif weekday == 5:  # Saturday
            return expected_today - timedelta(days=2)  # Previous Thursday
        else:  # Sunday (6)
            return expected_today - timedelta(days=3)  # Previous Thursday

    async def is_run_due(self) -> bool:
        """Determines if a run is currently due or was missed while offline."""
        now = datetime.now(timezone.utc)
        expected_run_time = await self.get_expected_last_run(now)
        
        try:
            from tg_bridge import connect_db
            async with connect_db() as db:
                async with db.execute("SELECT last_run_time, active FROM swarm_tasks WHERE task_name = 'trend_scout'") as cursor:
                    row = await cursor.fetchone()
                    if not row:
                        return False
                    last_run_str, active = row
                    if not active:
                        return False
                    if not last_run_str:
                        return True
                    try:
                        # Parse ISO 8601, ensuring it's timezone-aware
                        last_run_time = datetime.fromisoformat(last_run_str.replace('Z', '+00:00'))
                        if last_run_time.tzinfo is None:
                            last_run_time = last_run_time.replace(tzinfo=timezone.utc)
                    except ValueError:
                        return True
                        
                    return last_run_time < expected_run_time
        except Exception as e:
            logger.error(f"Error checking if trend scout run is due: {e}")
            # Prevent infinite loops/spam on DB connection failures by returning False
            return False

    async def run_scout(self, scheduled_time: datetime) -> bool:
        """Executes the agy subprocess, updates state in DB, and delivers digests to Telegram."""
        timestamp = int(datetime.now(timezone.utc).timestamp())
        session_id = f"sess_trend_scout_{timestamp}"
        
        try:
            from tg_bridge import connect_db
            async with connect_db() as db:
                async with db.execute("SELECT task_id FROM swarm_tasks WHERE task_name = 'trend_scout'") as cursor:
                    row = await cursor.fetchone()
                    task_id = row[0] if row else 1
                
                started_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
                await db.execute("""
                    INSERT INTO swarm_run_sessions (session_id, task_id, status, started_at)
                    VALUES (?, ?, 'RUNNING', ?)
                """, (session_id, task_id, started_at))
                await db.commit()
        except Exception as e:
            logger.error(f"Failed to initialize scheduler run session: {e}")
            task_id = 1

        prompt = """
        You are a highly skilled tech trend researcher. Your task is to perform an automated daily scout across the latest technology news, releases, and repositories using Tavily MCP and Exa MCP.
        
        Specifically search for updates, announcements, and news on these official developer blogs and sources:
        - Anthropic: https://www.anthropic.com/news
        - OpenAI: https://openai.com/news/ and https://openai.com/blog/
        - Google AI/DeepMind: https://deepmind.google/blog/ and https://blog.google/technology/ai/
        - Groq: https://groq.com/blog/ and https://groq.com/newsroom/
        - Alibaba Qwen: https://qwen.ai/blog
        - Moonshot AI (Kimi): https://platform.moonshot.cn/blog
        - DeepSeek: https://platform.deepseek.com/api-docs/changelog or official deepseek updates
        - Meta AI: https://ai.meta.com/blog/
        - Mistral AI: https://mistral.ai/news/
        - Hacker News: news.ycombinator.com (latest developer discussion on AI agents and model releases)
        - Academic papers: arxiv.org (specifically machine learning/AI categories)
        - GitHub: github.com (trending AI agent frameworks, multi-agent systems, and model repositories)
        
        Strict Token Compression & Cost Saving Rules (Caveman Mode):
        - Apply 'caveman' style (ultra compression) to all search planning, Q&A reasoning steps, and the final output report to minimize token consumption under subscription limits.
        - Eliminate all fluff, articles (a, an, the), pleasantries, introductory remarks, and conversational filler from thoughts and outputs.
        - Keep the generated summaries ultra-terse, factual, and strictly technical.
        - Use shorthand causal operators (e.g. X -> Y) and compressed notation where possible.
        - Preserving full technical accuracy is mandatory.
        
        Ensure to:
        - Use Tavily MCP for broad news, tech trends, and context searches.
        - Use Exa MCP for neural search and filtering specifically restricted to the target domains listed above (using domain inclusion where appropriate).
        - Limit the search gathering to a maximum of 50 total source URLs across all queries to optimize token usage.
        - Deduplicate news and filter out noise (such as general programming tutorials or off-topic tech news).
        - For each trend, provide a brief summary (why it matters) and a direct URL to the source.
        - Format the response in clean Markdown (using bold, italic, and bulleted lists).
        - Restrict the total length to 4000 characters.
        """
        
        env = {
            **os.environ,
            "HOME": "/root",
            "GEMINI_CLI_TRUST_WORKSPACE": "true"
        }
        
        if "TAVILY_API_KEY" not in env:
            env["TAVILY_API_KEY"] = os.environ.get("TAVILY_API_KEY", "")
        if "EXA_API_KEY" not in env:
            env["EXA_API_KEY"] = os.environ.get("EXA_API_KEY", "")

        from tg_bridge import _resolve_cli, connect_db
        cli = _resolve_cli()
        
        kwargs = {}
        if sys.platform != "win32":
            kwargs["preexec_fn"] = os.setpgrp
            
        try:
            proc = await asyncio.create_subprocess_exec(
                cli, "-p", prompt,
                env=env,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
                **kwargs
            )
            
            out, err = await asyncio.wait_for(proc.communicate(), timeout=300)
            returncode = proc.returncode
            
            raw_output = out.decode("utf-8", errors="replace").strip()
            stderr_output = err.decode("utf-8", errors="replace").strip()
            
            if returncode != 0:
                error_message = f"Process failed with code {returncode}\n{stderr_output}"
                await self._log_session_run(session_id, "FAILED", None, error_message, scheduled_time)
            else:
                lines = raw_output.splitlines()
                filtered_lines = []
                for line in lines:
                    lower_line = line.lower()
                    if "ripgrep" in lower_line or "warning:" in lower_line or "\x1b[" in line:
                        continue
                    filtered_lines.append(line)
                
                filtered_output = "\n".join(filtered_lines).strip()
                await self._send_to_telegram(filtered_output)
                
                summary_snippet = filtered_output[:200] + "..." if len(filtered_output) > 200 else filtered_output
                await self._log_session_run(session_id, "SUCCESS", summary_snippet, None, scheduled_time)
                
            return returncode == 0
                
        except asyncio.TimeoutError:
            import signal
            if sys.platform != "win32":
                try:
                    os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
                except Exception:
                    pass
            await self._log_session_run(session_id, "FAILED", None, "Subprocess execution timed out after 300 seconds", scheduled_time)
            return False
        except Exception as e:
            await self._log_session_run(session_id, "FAILED", None, f"Exception occurred: {str(e)}", scheduled_time)
            return False

    async def _send_to_telegram(self, markdown_text: str):
        from tg_bridge import md_to_tg_html, strip_markdown
        html_text = md_to_tg_html(markdown_text)
        
        for chat_id in self.allowed_users:
            try:
                await self.bot.send_message(chat_id=chat_id, text=html_text, parse_mode="HTML")
            except Exception as e:
                logger.warning(f"Failed to send HTML to {chat_id}, falling back to plain text: {e}")
                plain_text = strip_markdown(markdown_text)
                try:
                    await self.bot.send_message(chat_id=chat_id, text=plain_text)
                except Exception as ex:
                    logger.error(f"Failed to send plain text to {chat_id}: {ex}")

    async def _log_session_run(self, session_id: str, status: str, summary: str | None, error_message: str | None, scheduled_time: datetime | None = None, task_name: str = 'trend_scout'):
        try:
            from tg_bridge import connect_db
            finished_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
            async with connect_db() as db:
                async with db.execute("SELECT task_id FROM swarm_run_sessions WHERE session_id = ?", (session_id,)) as cursor:
                    row = await cursor.fetchone()
                    task_id = row[0] if row else None
                
                await db.execute("""
                    UPDATE swarm_run_sessions
                    SET status = ?, finished_at = ?, digest_summary = ?
                    WHERE session_id = ?
                """, (status, finished_at, summary or error_message, session_id))
                
                if scheduled_time and task_id:
                    iso_time = scheduled_time.strftime("%Y-%m-%dT%H:%M:%SZ")
                    await db.execute("""
                        UPDATE swarm_tasks
                        SET last_run_time = ?
                        WHERE task_id = ?
                    """, (iso_time, task_id))
                
                if task_name == 'trend_scout':
                    # Backwards-compatibility log
                    await db.execute(
                        "INSERT INTO trend_scout_runs (status, summary, error_message) VALUES (?, ?, ?)",
                        (status, summary, error_message)
                    )
                await db.commit()
        except Exception as e:
            logger.error(f"Failed to log run session: {e}")

    async def get_expected_last_blog_run(self, now: datetime) -> datetime:
        """Returns the datetime of the last expected 02:00 UTC run (Tuesday or Friday) relative to current time."""
        expected_today = datetime(now.year, now.month, now.day, 2, 0, 0, tzinfo=timezone.utc)
        weekday = now.weekday()  # 0=Monday, 1=Tuesday, 2=Wednesday, 3=Thursday, 4=Friday, 5=Saturday, 6=Sunday
        
        if weekday == 1:  # Tuesday
            if now >= expected_today:
                return expected_today
            else:
                return expected_today - timedelta(days=4)  # Previous Friday
        elif weekday == 2:  # Wednesday
            return expected_today - timedelta(days=1)  # Previous Tuesday
        elif weekday == 3:  # Thursday
            return expected_today - timedelta(days=2)  # Previous Tuesday
        elif weekday == 4:  # Friday
            if now >= expected_today:
                return expected_today
            else:
                return expected_today - timedelta(days=3)  # Previous Tuesday
        elif weekday == 5:  # Saturday
            return expected_today - timedelta(days=1)  # Previous Friday
        elif weekday == 6:  # Sunday
            return expected_today - timedelta(days=2)  # Previous Friday
        else:  # Monday (0)
            return expected_today - timedelta(days=3)  # Previous Friday

    async def is_blog_writer_due(self) -> bool:
        """Determines if a wordpress_blog_writer run is currently due."""
        now = datetime.now(timezone.utc)
        expected_run_time = await self.get_expected_last_blog_run(now)
        
        try:
            from tg_bridge import connect_db
            async with connect_db() as db:
                async with db.execute("SELECT last_run_time, active FROM swarm_tasks WHERE task_name = 'wordpress_blog_writer'") as cursor:
                    row = await cursor.fetchone()
                    if not row:
                        return False
                    last_run_str, active = row
                    if not active:
                        return False
                    if not last_run_str:
                        return True
                    try:
                        last_run_time = datetime.fromisoformat(last_run_str.replace('Z', '+00:00'))
                        if last_run_time.tzinfo is None:
                            last_run_time = last_run_time.replace(tzinfo=timezone.utc)
                    except ValueError:
                        return True
                        
                    return last_run_time < expected_run_time
        except Exception as e:
            logger.error(f"Error checking if blog writer run is due: {e}")
            return False

    async def run_blog_writer(self, scheduled_time: datetime) -> bool:
        """Executes the agy subprocess for wordpress_blog_writer task, updates state in DB, and delivers digests to Telegram."""
        timestamp = int(datetime.now(timezone.utc).timestamp())
        session_id = f"sess_blog_writer_{timestamp}"
        
        workspace_path = "/root/workspaces/telegram_pro_import"
        
        try:
            from tg_bridge import connect_db
            async with connect_db() as db:
                async with db.execute("SELECT task_id FROM swarm_tasks WHERE task_name = 'wordpress_blog_writer'") as cursor:
                    row = await cursor.fetchone()
                    task_id = row[0] if row else 2
                
                started_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
                await db.execute("""
                    INSERT INTO swarm_run_sessions (session_id, task_id, status, started_at)
                    VALUES (?, ?, 'RUNNING', ?)
                """, (session_id, task_id, started_at))
                await db.commit()
        except Exception as e:
            logger.error(f"Failed to initialize blog writer session: {e}")
            task_id = 2

        # Ensure we have a clean cloned workspace before running
        try:
            import shutil
            wpath = Path(workspace_path)
            if wpath.exists():
                shutil.rmtree(wpath)
            wpath.parent.mkdir(parents=True, exist_ok=True)
            
            git_proc = await asyncio.create_subprocess_exec(
                "git", "clone", "--depth", "1", "https://github.com/Azhy34/telegram_pro_import.git", workspace_path,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE
            )
            _, git_err = await git_proc.communicate()
            if git_proc.returncode != 0:
                logger.error(f"Failed to clone telegram_pro_import for scheduled run: {git_err.decode(errors='replace')}")
            else:
                logger.info(f"Successfully cloned telegram_pro_import to {workspace_path}")
        except Exception as clone_err:
            logger.error(f"Error preparing workspace for scheduled run: {clone_err}")

        prompt = """
        You are the Orchestrator (CEO) of the Antigravity Swarm.
        MISSION: Execute the scheduled blog writing and publishing pipeline for proimport-shop.ru.
        
        Workdir: /root/workspaces/telegram_pro_import
        
        To prevent memory/context overload, execute the pipeline step-by-step using local CLI tools:
        
        1. Read the next pending article from articles_queue.json. If none, exit. Set status to 'processing' and save.
        2. Run the Analyst script to select focus and secondary keywords:
           python3 scripts/agents/run_seo_analyst.py --topic "[topic]"
           (This saves keywords to data/keywords.json).
        3. Run the Copywriter script to draft the article in markdown:
           python3 scripts/agents/run_copywriter.py --post [post_file] --keywords data/keywords.json --output articles/[slug].md
        4. Run the HTML Designer script to compile the markdown to final HTML:
           python3 scripts/agents/run_html_designer.py --markdown articles/[slug].md --output articles/[slug].html
        5. Run 'python3 scripts/verify_article.py articles/[slug]' as a programmatic check.
        6. If verification fails, run the repair script to fix issues in-place:
           python3 scripts/agents/run_repair.py --file [failed_file] --errors "[errors]"
        7. Prepend the new blog card in articles/blog-page.html.
        8. Commit and push all changes (including local WebP assets) to the main branch, set status to 'published' in articles_queue.json, and push.
        
        When a command runs in the background, you MUST yield and wait for its completion message.
        Once the task completion message is received, you MUST proceed to the next step of the pipeline immediately.
        Do NOT exit or stop execution until the entire pipeline is complete (Step 8 is finished).
        
        Apply 'caveman' full mode. Keep responses concise but always state your action and call the next tool.
        Ensure isolation: do NOT modify files outside articles/, posts/, and articles_queue.json.
        """
        
        env = {
            **os.environ,
            "HOME": "/root",
            "GEMINI_CLI_TRUST_WORKSPACE": "true"
        }

        from tg_bridge import _resolve_cli, connect_db
        cli = _resolve_cli()
        
        kwargs = {}
        if sys.platform != "win32":
            kwargs["preexec_fn"] = os.setpgrp
            
        success = False
        try:
            proc = await asyncio.create_subprocess_exec(
                cli, "-p", prompt,
                env=env,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
                **kwargs
            )
            
            out, err = await asyncio.wait_for(proc.communicate(), timeout=600)
            returncode = proc.returncode
            
            raw_output = out.decode("utf-8", errors="replace").strip()
            stderr_output = err.decode("utf-8", errors="replace").strip()
            
            if returncode != 0:
                error_message = f"Process failed with code {returncode}\n{stderr_output}"
                await self._log_session_run(session_id, "FAILED", None, error_message, scheduled_time, 'wordpress_blog_writer')
            else:
                lines = raw_output.splitlines()
                filtered_lines = []
                for line in lines:
                    lower_line = line.lower()
                    if "ripgrep" in lower_line or "warning:" in lower_line or "\x1b[" in line:
                        continue
                    filtered_lines.append(line)
                
                filtered_output = "\n".join(filtered_lines).strip()
                await self._send_to_telegram(filtered_output)
                
                summary_snippet = filtered_output[:200] + "..." if len(filtered_output) > 200 else filtered_output
                await self._log_session_run(session_id, "SUCCESS", summary_snippet, None, scheduled_time, 'wordpress_blog_writer')
                success = True
                
        except asyncio.TimeoutError:
            import signal
            if sys.platform != "win32":
                try:
                    os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
                except Exception:
                    pass
            await self._log_session_run(session_id, "FAILED", None, "Execution timed out after 600 seconds", scheduled_time, 'wordpress_blog_writer')
        except Exception as e:
            await self._log_session_run(session_id, "FAILED", None, f"Exception occurred: {str(e)}", scheduled_time, 'wordpress_blog_writer')
        finally:
            # Clean up workspace folder to avoid VPS cluttering
            try:
                wpath = Path(workspace_path)
                if wpath.exists():
                    shutil.rmtree(wpath)
                    logger.info(f"Workspace cleaned up after scheduled run: {workspace_path}")
            except Exception as cleanup_err:
                logger.error(f"Failed to clean up workspace after scheduled run: {cleanup_err}")
                
        return success

    async def scheduler_loop(self, stop_event: asyncio.Event):
        """Infinite loop checking every 60 seconds if a task is due."""
        while not stop_event.is_set():
            try:
                if await self.is_run_due():
                    logger.info("Trend scout run is due, executing...")
                    now = datetime.now(timezone.utc)
                    expected_time = await self.get_expected_last_run(now)
                    await self.run_scout(expected_time)
            except Exception as e:
                logger.error(f"Error in trend scout scheduler loop: {e}")
                
            try:
                if await self.is_blog_writer_due():
                    logger.info("WordPress blog writer run is due, executing...")
                    now = datetime.now(timezone.utc)
                    expected_time = await self.get_expected_last_blog_run(now)
                    await self.run_blog_writer(expected_time)
            except Exception as e:
                logger.error(f"Error in blog writer scheduler loop: {e}")
                
            try:
                await asyncio.wait_for(stop_event.wait(), timeout=60.0)
            except asyncio.TimeoutError:
                continue


Youez - 2016 - github.com/yon3zu
LinuXploit