commit ec76ef64c8dc1d5b367610a7fc767b6541870640 Author: kenilkb Date: Wed Jul 1 18:00:54 2026 +0530 Initial diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..75f4e5d --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +FACTORS_EMAIL=natesh.krishnan@yourFirm.io +FACTORS_PASSWORD=Password@123 +SMTP_HOST=mail.tech4biz.org +SMTP_PORT=587 +SENDER_EMAIL=noreply@tech4biz.org +SENDER_PASSWORD=Password@123 +RECIPIENT_EMAILS=kenil@tech4biz.org, contact@tech4biz.io +HEADLESS=true \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb38501 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +venv +__pycache__ +.env +test_navigation.py +tmp_download +scratch +data +*.log +*.zip diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..99ee88c --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,482 @@ +# Factors.AI — Account Export Automation +### Automatic daily email alerts when new companies visit your website + +--- + +## What does this tool do? + +Every time you run it, this tool does **four things automatically** — no clicking, no manual work: + +``` +1. LOGS IN → Opens Factors.AI in the background and signs in with your credentials +2. DOWNLOADS → Goes to Account Profiles, clicks the export button, downloads the CSV +3. COMPARES → Checks: are there any new company domains that weren't in the last export? +4. EMAILS → If yes, sends a formatted HTML email to everyone on your list + If no new entries → does nothing (no email sent) +``` + +After every run, the fresh CSV is saved as `data/last_export.csv` — that becomes the baseline for the next comparison. + +--- + +## What the email looks like + +**Subject line examples:** + +| Situation | Subject | +|---|---| +| Very first run ever | `[Factors.AI] First Run — 483 Accounts Exported (2026-07-01)` | +| New companies found | `[Factors.AI] 7 New Accounts Detected (2026-07-01)` | +| Nothing new | *(no email sent at all)* | + +**Email body:** A styled HTML table with one row per new company, showing: + +| Account Domain | Company Name | Company Industry | Company Employee Range | Company Annual Revenue | Last Activity | +|---|---|---|---|---|---| +| dell.com | Dell | Internet Software & Services | 100K+ | $102,300,000,000 | 2026-07-01 04:48:32 | +| adobe.com | Adobe | Internet Software & Services | 10K-50K | $19,409,000,000 | 2026-06-18 11:29:45 | + +--- + +## How "new entry" is decided + +The tool uses **Account Domain** (the website address, e.g. `dell.com`) as the unique identifier for each company. + +- A company is **new** if its domain appears in today's export but was **not** in `data/last_export.csv` +- The comparison is case-insensitive (`Dell.com` = `dell.com`) +- Only `data/last_export.csv` is kept — no growing archive, no extra storage + +--- + +## Files in this folder + +``` +factors_automation/ +│ +├── main.py ← The only file you ever run ("python main.py") +├── scraper.py ← Controls the Chrome browser (login, navigate, download) +├── comparator.py ← Compares today's CSV with last_export.csv +├── mailer.py ← Builds the HTML email and sends it via SMTP +├── config.py ← All your settings live here (reads from .env) +│ +├── requirements.txt ← List of Python packages the tool needs +├── .env.example ← Template — copy this to .env and fill in your details +├── .env ← YOUR private settings (create this yourself — never share it) +│ +├── data/ +│ └── last_export.csv ← Auto-created after first run. This is the comparison baseline. +│ +├── tmp_download/ ← Temporary staging folder. Chrome downloads here first. +│ Cleaned automatically before each run. +│ Error screenshots are saved here if something goes wrong. +│ +└── automation.log ← Full log of every run (auto-created). Check this if anything fails. +``` + +> **Do not delete** `data/last_export.csv` — it is the memory of the tool. +> If you delete it, the next run will treat every account as new and email the full list. + +--- + +## Prerequisites — what you need before setup + +You need **three things** installed on the computer that will run this: + +### 1. Python 3.10 or newer +Check your version by opening a terminal and typing: +``` +python --version +``` +If it shows `Python 3.10.x` or higher, you're good. +If not, download from: https://www.python.org/downloads/ + +### 2. Google Chrome browser +The tool controls Chrome in the background. Download from: https://www.google.com/chrome/ + +> The matching ChromeDriver (what lets Python control Chrome) is downloaded **automatically** the first time you run the tool. You do not need to install it yourself. + +### 3. A Gmail account to send from (recommended) +Any Gmail account works. You will need to generate an **App Password** for it (explained in the setup below). You cannot use your regular Gmail password. + +--- + +## Setup — step by step + +### Step 1 — Download and unzip the project + +Unzip `factors_automation.zip` somewhere on your computer, for example: +- **Mac/Linux:** `~/factors_automation/` +- **Windows:** `C:\factors_automation\` + +### Step 2 — Open a terminal in that folder + +- **Mac:** Right-click the folder → "New Terminal at Folder" +- **Windows:** Open the folder in File Explorer → click the address bar → type `cmd` → press Enter + +### Step 3 — Create a virtual environment + +A virtual environment keeps this tool's packages separate from everything else on your computer. + +```bash +python -m venv venv +``` + +Then activate it: + +```bash +# Mac / Linux: +source venv/bin/activate + +# Windows: +venv\Scripts\activate +``` + +You will see `(venv)` appear at the start of your terminal prompt. This means it is active. + +### Step 4 — Install the required packages + +```bash +pip install -r requirements.txt +``` + +This downloads and installs four packages: +- **selenium** — controls the Chrome browser +- **webdriver-manager** — automatically downloads the right ChromeDriver +- **pandas** — reads and compares CSV files +- **python-dotenv** — reads your settings from the `.env` file + +### Step 5 — Create your `.env` settings file + +Copy the template: + +```bash +# Mac / Linux: +cp .env.example .env + +# Windows: +copy .env.example .env +``` + +Now open the `.env` file in any text editor (Notepad, TextEdit, VS Code, etc.) and fill in your real values. Here is what each line means: + +``` +FACTORS_EMAIL=natesh.krishnan@yourcompany.io +``` +The email address used to log in to app.factors.ai. + +``` +FACTORS_PASSWORD=YourActualPassword +``` +The password for that Factors.AI account. + +``` +SMTP_HOST=smtp.gmail.com +``` +Leave this as-is if you are using Gmail. Change only if you use a different email provider (Outlook = `smtp.office365.com`, etc.). + +``` +SMTP_PORT=587 +``` +Leave this as-is. Port 587 is the standard secure email port (STARTTLS). Works with all major email providers. + +``` +SENDER_EMAIL=your_sender@gmail.com +``` +The Gmail address the alerts will be sent **from**. + +``` +SENDER_PASSWORD=xxxx xxxx xxxx xxxx +``` +**This is NOT your regular Gmail password.** You must generate an **App Password** — see the section below. + +``` +RECIPIENT_EMAILS=person1@company.com,person2@company.com +``` +Everyone who should receive the alert emails. Separate multiple addresses with commas. No spaces around the commas. + +``` +HEADLESS=true +``` +`true` means Chrome runs invisibly in the background (recommended for scheduled/daily runs). +`false` means you can watch Chrome open and do its work on screen (useful when testing or debugging). + +--- + +### How to generate a Gmail App Password + +Gmail does not allow scripts to log in with your regular password. You must create a special one-time App Password. + +1. Go to your Google Account: https://myaccount.google.com/ +2. Click **Security** in the left sidebar +3. Under "How you sign in to Google", click **2-Step Verification** and enable it if not already on +4. Go back to Security → scroll down → click **App Passwords** +5. Under "Select app" choose **Mail** → under "Select device" choose **Other** → type `FactorsAI Bot` +6. Click **Generate** +7. Google shows a 16-character code like `abcd efgh ijkl mnop` +8. Copy it (including spaces) into your `.env` file as `SENDER_PASSWORD` + +> This App Password only lets the script send emails. It cannot access your Gmail inbox or change your account. + +--- + +## Choosing which columns to export + +By default (`FIELDS_TO_SELECT = None` in `config.py`), the tool keeps whichever columns are **already checked** in the Factors.AI export modal — which is the default 5: + +- Company Name +- Company Industry +- Company Employee Range +- Company Annual Revenue +- Last Activity + +The downloaded CSV will also always include **Account Domain** as the first column (this is always exported by Factors.AI and is used as the unique comparison key). + +### To change which columns are exported + +Open `config.py` in a text editor. Find this section near the bottom: + +```python +FIELDS_TO_SELECT: list[str] | None = None # None = use defaults +``` + +Change it to a list of the fields you want. For example, to export only Company Name, Industry, and Last Activity: + +```python +FIELDS_TO_SELECT = [ + "$6Signal_name", + "$6Signal_industry", + "last_activity", +] +``` + +**Full list of available field codes:** + +| Code | Column name in CSV | +|---|---| +| `"$6Signal_name"` | Company Name | +| `"$6Signal_industry"` | Company Industry | +| `"$6Signal_employee_range"` | Company Employee Range | +| `"$6Signal_annual_revenue"` | Company Annual Revenue | +| `"last_activity"` | Last Activity | +| `"$tag_hidden"` | Tags Hidden | +| `"$latest_source"` | Account Latest Source | +| `"$latest_campaign"` | Account Latest Campaign | +| `"$initial_campaign"` | Account First Campaign | +| `"$account_activity_url"` | Account Activity URL | +| `"$domain_name"` | Company ID | +| `"$latest_page_url"` | Account Latest Page URL | + +When you set `FIELDS_TO_SELECT` to a list, the tool first clicks "Clear All" in the modal, then checks exactly the fields you listed. Setting it back to `None` restores the default behaviour. + +--- + +## Running the tool + +Make sure `(venv)` is active in your terminal before running. + +### Normal run (invisible browser, sends real email) +```bash +python main.py +``` + +### Watch mode — opens real Chrome window (good for testing) +```bash +python main.py --visible +``` +Use this the first time you run it to verify everything works correctly. + +### Dry run — downloads and compares, but does NOT send any email +```bash +python main.py --dry-run +``` +The new entries are printed to the terminal instead. Useful for checking what would have been emailed. + +### Combine both flags +```bash +python main.py --visible --dry-run +``` + +--- + +## What happens during a run (step by step) + +When you run `python main.py`, here is exactly what happens internally: + +**Step 1 — Download CSV** +- Chrome launches (invisibly unless `--visible`) +- Opens `https://app.factors.ai/` +- Enters email and password from your `.env` +- Waits up to 30 seconds for login to complete +- Finds and clicks the Account Profiles page in the sidebar navigation. If that fails, it tries five known URL patterns (`/accounts`, `/account-profiles`, `/accounts/profiles`, `/analytics/accounts`, `/v2/accounts`) +- Clicks the download (↓) toolbar icon +- The export modal appears — columns are selected (or kept as default) +- Clicks "Export CSV" +- Waits up to 90 seconds for the file to appear in `tmp_download/` +- Chrome closes + +**Step 2 — Compare** +- Loads `tmp_download/.csv` and `data/last_export.csv` +- Compares every row's **Account Domain** (first column) +- Identifies domains in the new file that were not in the old file +- If `data/last_export.csv` does not exist → all rows are treated as new (first run) + +**Step 3 — Email** *(skipped if zero new entries and not first run)* +- Builds an HTML email with a table of new rows +- Connects to `smtp.gmail.com:587` via STARTTLS +- Logs in with `SENDER_EMAIL` and `SENDER_PASSWORD` +- Sends the email to all `RECIPIENT_EMAILS` + +**Step 4 — Save** +- Copies the downloaded CSV to `data/last_export.csv`, replacing the previous one +- This becomes the new baseline for the next run + +Everything is logged to `automation.log` in real time. + +--- + +## Scheduling — run automatically every day + +### Mac / Linux (using cron) + +Open the cron editor: +```bash +crontab -e +``` + +This runs every day at 01:30 UTC, which is 07:00 IST. (adjust the paths to match your actual folder): +``` +30 1 * * * /full/path/to/factors_automation/venv/bin/python /full/path/to/factors_automation/main.py +``` + +To find your full path, run this command inside the project folder: +```bash +pwd +``` + +Example result: `/home/natesh/factors_automation` + +So the cron line would be: +``` +0 8 * * * /home/natesh/factors_automation/venv/bin/python /home/natesh/factors_automation/main.py +``` + +### Windows (using Task Scheduler) + +1. Open **Task Scheduler** (search for it in the Start menu) +2. Click **Create Basic Task** on the right +3. Name: `Factors.AI Export` +4. Trigger: **Daily** at your preferred time +5. Action: **Start a program** +6. Program: `C:\factors_automation\venv\Scripts\python.exe` +7. Arguments: `C:\factors_automation\main.py` +8. Click **Finish** + +--- + +## Checking if a run succeeded + +Open `automation.log` in any text editor. A successful run looks like this: + +``` +2026-07-01 08:00:01 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:01 [INFO ] __main__ — Factors.AI Account Export Automation — START +2026-07-01 08:00:01 [INFO ] __main__ — headless=True dry_run=False +2026-07-01 08:00:01 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:01 [INFO ] __main__ — STEP 1: Downloading CSV … +2026-07-01 08:00:04 [INFO ] scraper — Opening https://app.factors.ai/ … +2026-07-01 08:00:09 [INFO ] scraper — Email entered. +2026-07-01 08:00:10 [INFO ] scraper — Password entered. +2026-07-01 08:00:11 [INFO ] scraper — Login form submitted. +2026-07-01 08:00:14 [INFO ] scraper — Logged in successfully. URL: https://app.factors.ai/accounts +2026-07-01 08:00:18 [INFO ] scraper — Download button clicked. +2026-07-01 08:00:21 [INFO ] scraper — 'Export CSV' button clicked. +2026-07-01 08:00:24 [INFO ] scraper — Download complete → tmp_download/accounts.csv +2026-07-01 08:00:24 [INFO ] scraper — Browser closed. +2026-07-01 08:00:24 [INFO ] __main__ — STEP 2: Comparing with previous export … +2026-07-01 08:00:24 [INFO ] comparator — Comparison complete: 7 new / 490 total (previous had 483 rows) +2026-07-01 08:00:24 [INFO ] __main__ — ✓ New entries detected (7 new) +2026-07-01 08:00:24 [INFO ] __main__ — STEP 3: Sending alert email … +2026-07-01 08:00:26 [INFO ] mailer — Email sent to: ['you@company.com'] | subject: [Factors.AI] 7 New Accounts Detected (2026-07-01) +2026-07-01 08:00:26 [INFO ] __main__ — STEP 4: Saving export as last_export.csv +2026-07-01 08:00:26 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:26 [INFO ] __main__ — Automation complete ✓ +``` + +--- + +## Troubleshooting + +### "Could not find email input on the login page" +The tool took a screenshot. Open `tmp_download/login_page_not_found.png` to see what the browser was looking at. + +Possible causes: +- Factors.AI is down or showing a maintenance page +- Your internet connection is slow (increase `_PAGE_LOAD_TIMEOUT` in `scraper.py` from `30` to `60`) +- The login page layout changed (contact whoever maintains this tool) + +### "Still on login page after submission — check credentials" +Your `FACTORS_EMAIL` or `FACTORS_PASSWORD` in `.env` is wrong. Check and correct them. +Screenshot saved as `tmp_download/post_login_timeout.png`. + +### "Could not navigate to the Account Profiles page" +The tool could not find the Accounts section. Screenshot saved as `tmp_download/accounts_not_found.png`. + +Try running with `--visible` to watch what happens: +```bash +python main.py --visible --dry-run +``` + +### "CSV download did not complete within 90 seconds" +The download took too long. Possible causes: +- Very large export (many thousands of rows) +- Slow internet + +Open `scraper.py`, find the line `_DOWNLOAD_TIMEOUT = 90` near the top, and change `90` to `180`. + +### Gmail error: "Username and Password not accepted" +You used your regular Gmail password instead of an App Password. +Follow the **"How to generate a Gmail App Password"** steps above. + +### Gmail error: "SMTPAuthenticationError" +Same as above, or 2-Step Verification is not enabled on the Gmail account. + +### `SyntaxError` or `python: command not found` +Your Python version is older than 3.10. Check with `python --version` and upgrade at https://www.python.org/downloads/ + +### The `(venv)` prefix disappeared from my terminal +Your virtual environment is no longer active. Reactivate it: +```bash +# Mac / Linux: +source venv/bin/activate + +# Windows: +venv\Scripts\activate +``` + +--- + +## Security notes + +- Your `.env` file contains passwords. **Never share it, email it, or commit it to Git.** +- The `.env.example` file contains only placeholder text and is safe to share. +- The Gmail App Password only allows the script to send email. It cannot read your inbox, delete messages, or change your Google account settings. +- `data/last_export.csv` contains company names and domains from Factors.AI. Treat it with the same care as any customer data. + +--- + +## Quick-reference card + +| Task | Command | +|---|---| +| Normal run (background, sends email) | `python main.py` | +| Watch browser work on screen | `python main.py --visible` | +| Test without sending email | `python main.py --dry-run` | +| Watch + test, no email | `python main.py --visible --dry-run` | + +| File | Purpose | +|---|---| +| `.env` | Your private credentials and settings | +| `config.py` | Export field selection and advanced settings | +| `data/last_export.csv` | Comparison baseline — do not delete | +| `automation.log` | Full run history — check when something fails | +| `tmp_download/*.png` | Error screenshots — check when a step fails | \ No newline at end of file diff --git a/comparator.py b/comparator.py new file mode 100644 index 0000000..06c3dc7 --- /dev/null +++ b/comparator.py @@ -0,0 +1,73 @@ +""" +comparator.py — Compares two Account Profiles CSVs and returns new entries. + +Logic +───── +• Unique key : "Account Domain" (the website domain, e.g. dell.com) +• New entry : a domain present in the NEW csv that is absent in the OLD csv. +• If OLD csv is missing (first ever run) → every row counts as "new". +""" +import logging +from pathlib import Path + +import pandas as pd + +logger = logging.getLogger(__name__) + +# Column used to identify a unique account across exports +_UNIQUE_KEY = "Account Domain" + + +def _load(path: str) -> pd.DataFrame: + return pd.read_csv(path, dtype=str).fillna("-").rename( + columns=lambda c: c.strip() + ) + + +def find_new_entries( + new_csv: str, + old_csv: str | None, +) -> tuple[pd.DataFrame, bool]: + """ + Parameters + ---------- + new_csv : path to the freshly downloaded CSV. + old_csv : path to the previously stored CSV, or None. + + Returns + ------- + (new_entries_df, is_first_run) + new_entries_df — DataFrame of rows that are brand-new. + is_first_run — True when there was no previous CSV to compare against. + """ + new_df = _load(new_csv) + + if _UNIQUE_KEY not in new_df.columns: + # The downloaded CSV might have a slightly different column order; + # treat the first column as the domain key. + logger.warning( + f"Column '{_UNIQUE_KEY}' not found. " + f"Available columns: {list(new_df.columns)}" + ) + first_col = new_df.columns[0] + new_df.rename(columns={first_col: _UNIQUE_KEY}, inplace=True) + logger.warning(f"Using '{first_col}' as the unique key instead.") + + if old_csv is None or not Path(old_csv).exists(): + logger.info("No previous CSV found — treating ALL entries as new (first run).") + return new_df, True + + old_df = _load(old_csv) + old_keys = set( + old_df[_UNIQUE_KEY].str.strip().str.lower() + if _UNIQUE_KEY in old_df.columns + else old_df.iloc[:, 0].str.strip().str.lower() + ) + mask = ~new_df[_UNIQUE_KEY].str.strip().str.lower().isin(old_keys) + new_entries = new_df[mask].copy().reset_index(drop=True) + + logger.info( + f"Comparison complete: {len(new_entries)} new / " + f"{len(new_df)} total (previous had {len(old_df)} rows)" + ) + return new_entries, False diff --git a/config.py b/config.py new file mode 100644 index 0000000..09227dc --- /dev/null +++ b/config.py @@ -0,0 +1,78 @@ +""" +config.py — Central configuration for Factors.AI automation. +All secrets are loaded from .env (copy .env.example → .env and fill in values). +""" +from argparse import RawDescriptionHelpFormatter +import os +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +# ── Paths ───────────────────────────────────────────────────────────────────── +BASE_DIR = Path(__file__).resolve().parent +DATA_DIR = BASE_DIR / "data" +TEMP_DOWNLOAD_DIR = BASE_DIR / "tmp_download" +LAST_CSV_PATH = DATA_DIR / "last_export.csv" +LOG_PATH = BASE_DIR / "automation.log" + +DATA_DIR.mkdir(parents=True, exist_ok=True) +TEMP_DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) + +# ── Factors.AI credentials ──────────────────────────────────────────────────── +FACTORS_EMAIL = os.getenv("FACTORS_EMAIL", "Add_your_mail@env.com") +FACTORS_PASSWORD = os.getenv("FACTORS_PASSWORD", "Admin@123") +TARGET_PROJECT = os.getenv("TARGET_PROJECT", "Inbound-Tech4Biz") + +# ── Email / SMTP settings ───────────────────────────────────────────────────── +# For Gmail: enable 2FA, create an App Password and use it as SENDER_PASSWORD. +SMTP_HOST = os.getenv("SMTP_HOST", "smtp.zoho.com") +SMTP_PORT = int(os.getenv("SMTP_PORT", "587")) +SENDER_EMAIL = os.getenv("SENDER_EMAIL", "your_sender@gmail.com") +SENDER_PASSWORD = os.getenv("SENDER_PASSWORD", "your_app_password") + +# Comma-separated list of recipient emails +RECIPIENT_EMAILS: list[str] = [ + e.strip() + for e in os.getenv("RECIPIENT_EMAILS", "recipient@example.com").split(",") + if e.strip() +] + +# ── Fields to select in the CSV export modal ────────────────────────────────── +# Set to None → keep whatever boxes are pre-checked (default 5 fields). +# Set to list → clear all, then check exactly these field values. +# +# Known field values (from the modal checkboxes): +# "$6Signal_name" Company Name +# "$6Signal_industry" Company Industry +# "$6Signal_employee_range" Company Employee Range +# "$6Signal_annual_revenue" Company Annual Revenue +# "last_activity" Last Activity +# "$tag_hidden" Tags Hidden +# "$latest_source" Account Latest Source +# "$latest_campaign" Account Latest Campaign +# "$initial_campaign" Account First Campaign +# "$account_activity_url" Account Activity URL +# "$domain_name" Company ID +# "$latest_page_url" Account Latest Page URL +# +# FIELDS_TO_SELECT: list[str] | None = None # None = use defaults + +# Uncomment ↓ to specify exact fields instead of defaults: +FIELDS_TO_SELECT = [ +"Company Name", # "$6Signal_name", +"Company Industry", # "$6Signal_industry", +"Company Employee Range", # "$6Signal_employee_range", +"Company Annual Revenue", # "$6Signal_annual_revenue", +"Last Activity", # "last_activity", +"Company LinkedIn URL", # "$enriched_company_linkedin_url", +"Company Description", # "$enriched_company_description", +"Company Domain", # "$6Signal_domain", +"Company HQ Address", # "$6Signal_address", +"Company HQ City", # "$6Signal_city", +"Company Employee Count" # "$6Signal_employee_count", +] + +# ── Browser settings ────────────────────────────────────────────────────────── +# HEADLESS=false opens a real browser window (handy for debugging). +HEADLESS = os.getenv("HEADLESS", "true").lower() == "true" diff --git a/mailer.py b/mailer.py new file mode 100644 index 0000000..0c4d446 --- /dev/null +++ b/mailer.py @@ -0,0 +1,711 @@ +""" +mailer.py — Sends a polished HTML digest of new Factors.AI account entries, +with the full export attached as a CSV file. + +Design notes +------------ +* Card-per-account layout instead of a single wide table. An 11-column table + cannot fit inside an email viewport on any client, and Outlook desktop + (Word rendering engine) cannot horizontally scroll — a wide table there + just renders broken/clipped. Stacked cards render correctly everywhere, + including Outlook, Gmail, Apple Mail and mobile clients, with zero + horizontal overflow. +* All styling is inline (email clients strip most + + + + + + + + +
+ + + + + + + + + + + + + + {f'" if False else ""} + {attachment_html} + + + + + + + + + + + + +
+ +""" + + +def _build_plain_text(df: pd.DataFrame, is_first_run: bool, csv_filename: Optional[str]) -> str: + count = len(df) + lines = ["Factors.AI - Account Alerts", "=" * 32, ""] + if is_first_run: + lines.append(f"Baseline export - {count} account(s) currently in Factors.AI.") + else: + lines.append(f"{count} new account(s) detected since the last export.") + lines.append("") + + for _, row in df.iterrows(): + name = _clean(row.get(_PRIMARY_FIELD)) or _clean(row.get(_DOMAIN_FIELD)) or "Unknown company" + domain = _clean(row.get(_DOMAIN_FIELD)) + header = name + (f" ({domain})" if domain and domain != name else "") + lines.append(f"- {header}") + for field in [*_BADGE_FIELDS, *_GRID_FIELDS, _ADDRESS_FIELD]: + val = _clean(row.get(field)) + if val: + lines.append(f" {_display_label(field)}: {val}") + linkedin = _clean(row.get(_LINKEDIN_FIELD)) + if linkedin: + lines.append(f" LinkedIn: {linkedin}") + lines.append("") + + if csv_filename: + lines.append(f"Full export attached: {csv_filename}") + lines.append("") + lines.append("This is an automated message from the Factors.AI Account Export script.") + return "\n".join(lines) + + +# ── Attachment ──────────────────────────────────────────────────────────────── + +def _attach_csv(msg: MIMEMultipart, csv_path: Optional[str]) -> Optional[str]: + """Attach the exported CSV to the message. Returns the filename attached, or None.""" + if not csv_path: + logger.info("No downloaded_csv path provided — sending without an attachment.") + return None + + path = Path(csv_path) + if not path.is_file(): + logger.warning(f"downloaded_csv not found, skipping attachment: {path}") + return None + + try: + data = path.read_bytes() + except OSError as exc: + logger.warning(f"Could not read downloaded_csv ({path}): {exc} — skipping attachment.") + return None + + part = MIMEApplication(data, _subtype="csv") + part.add_header("Content-Disposition", "attachment", filename=path.name) + msg.attach(part) + logger.info(f"Attached CSV: {path.name} ({len(data):,} bytes)") + return path.name + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def send_alert( + smtp_host: str, + smtp_port: int, + sender_email: str, + sender_password: str, + recipients: Sequence[str], + new_entries: pd.DataFrame, + downloaded_csv: Optional[str] = None, + *, + is_first_run: bool = False, + skip_if_empty: bool = True, + use_ssl: bool = False, +) -> bool: + """ + Compose and send the Factors.AI account alert email, with the raw CSV export attached. + + Parameters + ---------- + smtp_host / smtp_port : SMTP server. Use use_ssl=True for port 465 (implicit TLS), + or leave use_ssl=False for STARTTLS (typically port 587). + sender_email : The From address. + sender_password : SMTP password / App Password. + recipients : List of To addresses. Must be non-empty. + new_entries : DataFrame of new rows to include in the digest. + downloaded_csv : Path to the full CSV export to attach. If missing/unreadable, + the email still sends without the attachment (logged as a warning). + is_first_run : True if no previous CSV existed (baseline export copy/subject). + skip_if_empty : If True (default) and new_entries is empty and this is not the + first run, no email is sent — avoids empty "0 new accounts" noise. + use_ssl : Use SMTP_SSL instead of STARTTLS. + + Returns + ------- + bool — True if an email was sent, False if sending was skipped (empty digest). + """ + if not recipients: + raise ValueError("recipients must contain at least one address") + if new_entries is None: + raise ValueError("new_entries cannot be None") + + count = len(new_entries) + if count == 0 and not is_first_run and skip_if_empty: + logger.info("No new accounts detected — skipping email (skip_if_empty=True).") + return False + + today = datetime.now().strftime("%Y-%m-%d") + subject = ( + f"[Factors.AI] Baseline Export — {count} Account{'s' if count != 1 else ''} ({today})" + if is_first_run + else f"[Factors.AI] {count} New Account{'s' if count != 1 else ''} Detected ({today})" + ) + + msg = MIMEMultipart("mixed") + msg["Subject"] = subject + msg["From"] = sender_email + msg["To"] = ", ".join(recipients) + msg["Date"] = formatdate(localtime=True) + msg["Message-ID"] = make_msgid() + + csv_filename = Path(downloaded_csv).name if downloaded_csv and Path(downloaded_csv).is_file() else None + + alt = MIMEMultipart("alternative") + alt.attach(MIMEText(_build_plain_text(new_entries, is_first_run, csv_filename), "plain", "utf-8")) + alt.attach(MIMEText(_build_html(new_entries, is_first_run, csv_filename), "html", "utf-8")) + msg.attach(alt) + + _attach_csv(msg, downloaded_csv) + + logger.info(f"Connecting to {smtp_host}:{smtp_port} ...") + try: + smtp_cls = smtplib.SMTP_SSL if use_ssl else smtplib.SMTP + with smtp_cls(smtp_host, smtp_port, timeout=30) as server: + server.ehlo() + if not use_ssl: + server.starttls() + server.ehlo() + server.login(sender_email, sender_password) + server.sendmail(sender_email, list(recipients), msg.as_string()) + except smtplib.SMTPException: + logger.exception(f"Failed to send Factors.AI alert email to {recipients}") + raise + + logger.info(f"Email sent to: {recipients} | subject: {subject}") + return True + + +def save_preview( + new_entries: pd.DataFrame, + output_path: str = "preview.html", + is_first_run: bool = False, + csv_filename: Optional[str] = "factors_ai_export.csv", +) -> str: + """Render the email to a local HTML file without sending anything — for quick design checks.""" + html_body = _build_html(new_entries, is_first_run, csv_filename) + Path(output_path).write_text(html_body, encoding="utf-8") + return output_path + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# """ +# mailer.py — Sends an HTML email listing new Factors.AI account entries. +# """ +# import logging +# import smtplib +# from datetime import datetime +# from email.mime.multipart import MIMEMultipart +# from email.mime.text import MIMEText + +# import pandas as pd + +# logger = logging.getLogger(__name__) + +# # Columns to show in the email table (in this order) +# _EMAIL_COLS = [ +# "Account Domain", +# "Company Name", +# "Company Industry", +# "Company Employee Range", +# "Company Annual Revenue", +# "Last Activity", +# ] + +# # ── HTML helpers ────────────────────────────────────────────────────────────── + +# _STYLES = { +# "body": "font-family:Arial,sans-serif;font-size:14px;color:#333;margin:0;padding:0;background:#f4f6f9", +# "wrapper": "max-width:900px;margin:32px auto;background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 2px 8px rgba(0,0,0,.12)", +# "header": "background:#4F46E5;padding:24px 32px", +# "h1": "color:#fff;margin:0;font-size:22px", +# "sub": "color:#c7d2fe;font-size:13px;margin:6px 0 0", +# "body_div": "padding:24px 32px", +# "table": "border-collapse:collapse;width:100%;font-size:13px", +# "th": "padding:10px 14px;background:#4F46E5;color:#fff;text-align:left;white-space:nowrap", +# "td_even": "padding:9px 14px;border-bottom:1px solid #eef0f3;background:#f9fafb", +# "td_odd": "padding:9px 14px;border-bottom:1px solid #eef0f3;background:#fff", +# "footer": "padding:16px 32px;background:#f4f6f9;font-size:11px;color:#9ca3af", +# } + + +# def _make_table(df: pd.DataFrame) -> str: +# cols = [c for c in _EMAIL_COLS if c in df.columns] +# if not cols: +# cols = list(df.columns) + +# headers = "".join( +# f"{c}" for c in cols +# ) + +# rows_html = "" +# for i, (_, row) in enumerate(df[cols].iterrows()): +# td_style = _STYLES["td_even"] if i % 2 == 0 else _STYLES["td_odd"] +# cells = "".join( +# f"{str(v)}" for v in row +# ) +# rows_html += f"{cells}" + +# tbl_style = _STYLES["table"] +# return ( +# f"" +# f"{headers}" +# f"{rows_html}" +# f"
" +# ) + + +# def _build_html(df: pd.DataFrame, is_first_run: bool) -> str: +# count = len(df) +# run_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +# if is_first_run: +# intro = ( +# "This is the first run of the automation — no previous " +# "export exists for comparison. All current accounts are listed below." +# ) +# else: +# intro = ( +# f"{count} new account{'s' if count != 1 else ''} " +# f"{'have' if count != 1 else 'has'} been detected since the last export." +# ) + +# table_html = _make_table(df) + +# return f""" +# +# +#
+ +#
+#

Factors.AI — Account Profiles Report

+#

Run time: {run_time}

+#
+ +#
+#

{intro}

+#

+# Total rows in this export: {count} +#

+#
+# {table_html} +#
+ +#
+# This is an automated message generated by the Factors.AI Account +# Export script. Do not reply to this email. +#
+ +#
+# +# +# """ + + +# # ── Public API ──────────────────────────────────────────────────────────────── + +# def send_alert( +# smtp_host: str, +# smtp_port: int, +# sender_email: str, +# sender_password: str, +# recipients: list[str], +# new_entries: pd.DataFrame, +# is_first_run: bool = False, +# downloaded_csv: str, +# ) -> None: +# """ +# Compose and send the alert email. + +# Parameters +# ---------- +# smtp_host / smtp_port : SMTP server (default: Gmail STARTTLS 587). +# sender_email : The From address. +# sender_password : SMTP password / App Password. +# recipients : List of To addresses. +# new_entries : DataFrame of new rows to include. +# is_first_run : True if no previous CSV existed. +# """ +# count = len(new_entries) +# today = datetime.now().strftime("%Y-%m-%d") + +# subject = ( +# f"[Factors.AI] First Run — {count} Accounts Exported ({today})" +# if is_first_run +# else f"[Factors.AI] {count} New Account{'s' if count != 1 else ''} Detected ({today})" +# ) + +# html_body = _build_html(new_entries, is_first_run) + +# msg = MIMEMultipart("alternative") +# msg["Subject"] = subject +# msg["From"] = sender_email +# msg["To"] = ", ".join(recipients) +# msg.attach(MIMEText(html_body, "html", "utf-8")) + +# logger.info(f"Connecting to {smtp_host}:{smtp_port} …") +# with smtplib.SMTP(smtp_host, smtp_port, timeout=30) as server: +# server.ehlo() +# server.starttls() +# server.login(sender_email, sender_password) +# server.sendmail(sender_email, recipients, msg.as_string()) + +# logger.info(f"Email sent to: {recipients} | subject: {subject}") diff --git a/main.py b/main.py new file mode 100644 index 0000000..0fe7533 --- /dev/null +++ b/main.py @@ -0,0 +1,136 @@ +""" +main.py — Entry point for the Factors.AI account-export automation. + +Usage +───── + python main.py # uses settings in config.py / .env + python main.py --visible # opens a real browser window (debug) + python main.py --dry-run # skip email; just download & compare + +Schedule (cron example — runs every day at 8 AM) +───────────────────────────────────────────────── + 0 8 * * * /path/to/venv/bin/python /path/to/factors_automation/main.py +""" +import argparse +import logging +import shutil +import sys +from pathlib import Path + +# ── Logging (set up before any other import) ────────────────────────────────── +from config import LOG_PATH # noqa: E402 – needs to be early + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)-8s] %(name)s — %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(str(LOG_PATH), encoding="utf-8"), + ], +) +logger = logging.getLogger(__name__) + +# ── Application imports ─────────────────────────────────────────────────────── +from config import ( # noqa: E402 + FACTORS_EMAIL, + FACTORS_PASSWORD, + TARGET_PROJECT, + TEMP_DOWNLOAD_DIR, + LAST_CSV_PATH, + SMTP_HOST, + SMTP_PORT, + SENDER_EMAIL, + SENDER_PASSWORD, + RECIPIENT_EMAILS, + FIELDS_TO_SELECT, + HEADLESS, +) +from scraper import FactorsAIScraper +from comparator import find_new_entries +from mailer import send_alert + + +# ── CLI arguments ───────────────────────────────────────────────────────────── + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Factors.AI account-export automation") + p.add_argument( + "--visible", + action="store_true", + help="Open a real browser window instead of running headless", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Download and compare but do NOT send the email", + ) + return p.parse_args() + + +# ── Main flow ───────────────────────────────────────────────────────────────── + +def main() -> None: + args = _parse_args() + headless = HEADLESS and not args.visible + + logger.info("=" * 65) + logger.info(" Factors.AI Account Export Automation — START") + logger.info(f" headless={headless} dry_run={args.dry_run}") + logger.info("=" * 65) + + # ── Step 1 ─ Download CSV via Selenium ──────────────────────────────────── + logger.info("STEP 1: Downloading CSV …") + scraper = FactorsAIScraper( + download_dir=str(TEMP_DOWNLOAD_DIR), + headless=headless, + ) + downloaded_csv = scraper.run( + email=FACTORS_EMAIL, + password=FACTORS_PASSWORD, + target_project=TARGET_PROJECT, + fields_to_select=FIELDS_TO_SELECT, + ) + logger.info(f" Downloaded → {downloaded_csv}") + + # ── Step 2 ─ Compare with last export ──────────────────────────────────── + logger.info("STEP 2: Comparing with previous export …") + last_csv = str(LAST_CSV_PATH) if LAST_CSV_PATH.exists() else None + new_entries, is_first_run = find_new_entries(downloaded_csv, last_csv) + + if new_entries.empty and not is_first_run: + logger.info(" ✓ No new entries found — nothing to mail.") + else: + count = len(new_entries) + tag = "(first run — all entries)" if is_first_run else f"({count} new)" + logger.info(f" ✓ New entries detected {tag}") + + # ── Step 3 ─ Send email ─────────────────────────────────────────────── + if args.dry_run: + logger.info("STEP 3: [DRY RUN] Email skipped.") + logger.info(new_entries.to_string()) + else: + logger.info("STEP 3: Sending alert email …") + send_alert( + smtp_host=SMTP_HOST, + smtp_port=SMTP_PORT, + sender_email=SENDER_EMAIL, + sender_password=SENDER_PASSWORD, + recipients=RECIPIENT_EMAILS, + new_entries=new_entries, + is_first_run=is_first_run, + downloaded_csv=downloaded_csv + ) + logger.info(" ✓ Email sent.") + + # ── Step 4 ─ Persist the new CSV as last_export.csv ───────────────────── + logger.info(f"STEP 4: Saving export as last_export.csv → {LAST_CSV_PATH}") + shutil.copy2(downloaded_csv, str(LAST_CSV_PATH)) + + logger.info("=" * 65) + logger.info(" Automation complete ✓") + logger.info("=" * 65) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aad0a21 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +selenium +webdriver-manager +pandas +python-dotenv \ No newline at end of file diff --git a/scraper.py b/scraper.py new file mode 100644 index 0000000..b5ab7ca --- /dev/null +++ b/scraper.py @@ -0,0 +1,833 @@ +""" +scraper.py — Selenium automation to log in to Factors.AI and download the + Account Profiles CSV. +""" +import glob +import logging +import os +import random +import time +from pathlib import Path + +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager +from selenium.common.exceptions import TimeoutException + +logger = logging.getLogger(__name__) + +_PAGE_LOAD_TIMEOUT = 30 # seconds to wait for page transitions +_ELEMENT_TIMEOUT = 20 # seconds to wait for elements +_DOWNLOAD_TIMEOUT = 90 # seconds to wait for file download + + +class FactorsAIScraper: + """Downloads the Account Profiles CSV export from app.factors.ai.""" + + def __init__(self, download_dir: str, headless: bool = True): + self.download_dir = str(Path(download_dir).resolve()) + self.headless = headless + self.driver: webdriver.Chrome | None = None + + # ── Driver setup ────────────────────────────────────────────────────────── + + def _setup_driver(self) -> None: + options = Options() + + if self.headless: + options.add_argument("--headless=new") + + # Use a persistent Chrome profile to keep cookies and session active + profile_dir = os.path.join(self.download_dir, "chrome_profile") + os.makedirs(profile_dir, exist_ok=True) + # Clean up stale chrome locks + lock_file = os.path.join(profile_dir, "SingletonLock") + if os.path.islink(lock_file) or os.path.exists(lock_file): + try: + os.remove(lock_file) + logger.info("Cleaned up stale SingletonLock from Chrome profile.") + except Exception as e: + logger.warning(f"Could not remove SingletonLock: {e}") + options.add_argument(f"--user-data-dir={profile_dir}") + + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-gpu") + options.add_argument("--window-size=1920,1080") + options.add_argument("--disable-extensions") + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_experimental_option("excludeSwitches", ["enable-automation"]) + options.add_experimental_option("useAutomationExtension", False) + + prefs = { + "download.default_directory": self.download_dir, + "download.prompt_for_download": False, + "download.directory_upgrade": True, + "safebrowsing.enabled": False, + "safebrowsing.disable_download_protection": True, + } + options.add_experimental_option("prefs", prefs) + + service = Service(ChromeDriverManager().install()) + self.driver = webdriver.Chrome(service=service, options=options) + self.driver.set_page_load_timeout(_PAGE_LOAD_TIMEOUT) + + # Enable downloads explicitly when headless (CDP call) + if self.headless: + try: + self.driver.execute_cdp_cmd( + "Page.setDownloadBehavior", + {"behavior": "allow", "downloadPath": self.download_dir}, + ) + except Exception: + pass # Older ChromeDriver builds may not support this + + # ── Helpers ─────────────────────────────────────────────────────────────── + + def _wait( + self, + by: str, + value: str, + timeout: int = _ELEMENT_TIMEOUT, + condition: str = "presence", + ): + ec_map = { + "presence": EC.presence_of_element_located, + "clickable": EC.element_to_be_clickable, + "visible": EC.visibility_of_element_located, + } + return WebDriverWait(self.driver, timeout).until( + ec_map[condition]((by, value)) + ) + + def _try_selectors(self, selectors: list[tuple], timeout: int = 8): + """Return the first element found from a list of (By.*, selector) pairs, waiting at most timeout seconds total.""" + deadline = time.time() + timeout + while True: + for by, sel in selectors: + try: + elements = self.driver.find_elements(by, sel) + for el in elements: + if el.is_displayed() and el.is_enabled(): + return el + except Exception: + pass + if time.time() > deadline: + break + time.sleep(0.2) + return None + + def _click(self, element) -> None: + """Click an element, falling back to JS click if intercepted.""" + try: + element.click() + except Exception as e: + logger.warning(f"Standard click failed ({e}), attempting JS click...") + self.driver.execute_script("arguments[0].click();", element) + + def _type_humanlike(self, element, text: str) -> None: + """Type text into an element with a short random delay between keystrokes.""" + element.click() + from selenium.webdriver.common.keys import Keys + try: + element.send_keys(Keys.CONTROL + "a") + element.send_keys(Keys.BACKSPACE) + except Exception: + element.clear() + + for char in text: + element.send_keys(char) + time.sleep(random.uniform(0.04, 0.12)) + + try: + self.driver.execute_script("arguments[0].dispatchEvent(new Event('input', { bubbles: true }));", element) + self.driver.execute_script("arguments[0].dispatchEvent(new Event('change', { bubbles: true }));", element) + except Exception: + pass + + # ── Login ───────────────────────────────────────────────────────────────── + + def login(self, email: str, password: str) -> None: + logger.info("Opening https://app.factors.ai/ …") + self.driver.get("https://app.factors.ai/") + time.sleep(3) + + # Check if we are already logged in + current_url = self.driver.current_url.lower() + if all(kw not in current_url for kw in ("login", "signin", "sign-in")): + logger.info("Already logged in (session restored from profile).") + return + + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + logger.info(f"Login attempt {attempt}/{max_attempts} …") + + # Find input fields + email_input = self._try_selectors([ + (By.CSS_SELECTOR, "input[name='email']"), + (By.CSS_SELECTOR, "input[type='email']"), + (By.XPATH, "//input[@placeholder[contains(translate(.,'EMAIL','email'),'email')]]"), + (By.XPATH, "//input[@autocomplete='username']"), + (By.XPATH, "//input[@autocomplete='email']"), + ], timeout=5) + + if email_input is None: + self._save_screenshot("login_page_not_found.png") + raise RuntimeError("Could not find email input on the login page.") + + self._type_humanlike(email_input, email) + logger.info("Email entered.") + + pwd_input = self._wait( + By.CSS_SELECTOR, "input[type='password']", timeout=10 + ) + self._type_humanlike(pwd_input, password) + logger.info("Password entered.") + + submit = self._try_selectors([ + (By.CSS_SELECTOR, "button[type='submit']"), + (By.XPATH, "//button[contains(translate(.,'LOGIN','login'),'login') or contains(translate(.,'SIGN IN','sign in'),'sign in')]"), + (By.CSS_SELECTOR, "button.ant-btn-primary"), + ], timeout=10) + + if submit is None: + raise RuntimeError("Could not find login submit button.") + + self._click(submit) + logger.info("Login form submitted.") + time.sleep(5) + + # Check if login failed (e.g. error message displayed) + try: + err = WebDriverWait(self.driver, 5).until( + EC.visibility_of_element_located(( + By.XPATH, + "//*[self::div or self::span or self::p or contains(@class,'error') or contains(@class,'message')][contains(text(), 'Login failed') or contains(text(), 'invalid') or contains(text(), 'incorrect')]" + )) + ) + logger.warning(f"Login failed message detected: {err.text}") + if attempt < max_attempts: + sleep_time = 45 * attempt + logger.info(f"Waiting {sleep_time} seconds before retrying login...") + time.sleep(sleep_time) + # Refresh the page to start clean + self.driver.get("https://app.factors.ai/") + time.sleep(3) + continue + else: + raise RuntimeError(f"Login rejected by Factors.AI after {max_attempts} attempts: {err.text}") + except TimeoutException: + # No failure message found, verify if we are logged in + current_url = self.driver.current_url.lower() + if all(kw not in current_url for kw in ("login", "signin", "sign-in")): + logger.info(f"Logged in successfully. URL: {self.driver.current_url}") + return + else: + # No error message, but still on login page. Wait and check again. + time.sleep(3) + if all(kw not in self.driver.current_url.lower() for kw in ("login", "signin", "sign-in")): + logger.info(f"Logged in successfully. URL: {self.driver.current_url}") + return + + raise RuntimeError("Still on login page after submission.") + + # ── Navigate to Account Profiles ────────────────────────────────────────── + + def _navigate_to_accounts(self) -> None: + """Find and open the Account Profiles page.""" + logger.info("Navigating to Account Profiles page …") + + # Derive base URL (e.g. https://app.factors.ai) from current URL + parts = self.driver.current_url.split("/") + base_url = "/".join(parts[:3]) # ['https:', '', 'app.factors.ai'] + + # 1. If we are already on the base URL, just wait for the download button to be present + from urllib.parse import urlparse + parsed = urlparse(self.driver.current_url) + path = parsed.path.rstrip("/") + if path in ("", "/", "/v2/accounts"): + logger.info("Already on the home/accounts URL. Waiting for the Accounts page to load...") + self._dismiss_upgrade_modal() + if self._download_button_present(timeout=25): + logger.info("Accounts page loaded successfully on the home page. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + + # 2. Check if the download button is already present (e.g. from a previous navigation) + self._dismiss_upgrade_modal() + if self._download_button_present(timeout=3): + logger.info("Already on the Account Profiles page. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + + # 3. Try clicking a sidebar / nav link first (fastest) + nav_candidates = [ + (By.ID, "fa-at-link--accounts"), + (By.XPATH, "//a[contains(translate(@href,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]"), + (By.XPATH, "//span[contains(translate(text(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]/ancestor::a"), + (By.XPATH, "//li[contains(translate(.,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]//a"), + ] + link = self._try_selectors(nav_candidates, timeout=5) + if link: + try: + self._click(link) + time.sleep(1) + if self._download_button_present(timeout=5): + logger.info(f"Accounts page found via nav. URL: {self.driver.current_url}. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + except Exception: + pass + + # 4. Try known URL patterns + for slug in [ + "/v2/accounts", + "/accounts", + "/", + ]: + url = base_url + slug + logger.info(f" Trying {url} …") + try: + self.driver.get(url) + if self._download_button_present(timeout=3): + logger.info(f"Accounts page found at: {url}. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + except Exception: + pass + + # Last resort: dump screenshot and raise + self._save_screenshot("accounts_not_found.png") + raise RuntimeError( + "Could not navigate to the Account Profiles page. " + "Screenshot saved → tmp_download/accounts_not_found.png" + ) + + def _download_button_present(self, timeout: int = 6) -> bool: + """Return True if the toolbar download button is visible on the page.""" + combined_selector = ( + "button#fa-at-btn--export-csv, " + "button[id*='export-csv'], " + "[class*='profileToolbarIconBtn'], " + "button[title*='Download'], " + "button[title*='Export']" + ) + try: + WebDriverWait(self.driver, timeout).until( + EC.presence_of_element_located((By.CSS_SELECTOR, combined_selector)) + ) + return True + except TimeoutException: + # Fallback to xpath check + try: + WebDriverWait(self.driver, 1).until( + EC.presence_of_element_located(( + By.XPATH, + "//button[.//svg[@data-icon='download'] or contains(@class,'profileToolbarIconBtn')]" + )) + ) + return True + except TimeoutException: + return False + + def _click_download_button(self) -> None: + logger.info("Clicking the download toolbar button …") + + for attempt in range(3): + self._dismiss_upgrade_modal() + + # Find all profile toolbar buttons + candidates = self.driver.find_elements(By.CSS_SELECTOR, "[class*='profileToolbarIconBtn']") + btn = None + for c in candidates: + try: + html = c.get_attribute("outerHTML") or "" + if "download" in html or "fa-download" in html: + if c.is_displayed() and c.is_enabled(): + btn = c + break + except Exception: + pass + + if btn is None: + logger.warning(f"Download button not found on attempt {attempt + 1}") + time.sleep(2) + continue + + try: + self.driver.execute_script("arguments[0].scrollIntoView(true);", btn) + time.sleep(0.5) + self.driver.execute_script("arguments[0].click();", btn) + logger.info(f"Download button clicked via JS (attempt {attempt + 1}).") + except Exception as e: + logger.warning(f"Failed to click button on attempt {attempt + 1}: {e}") + time.sleep(2) + continue + + time.sleep(2.0) + + modals = self.driver.find_elements(By.CSS_SELECTOR, ".ant-modal-content") + if any(m.is_displayed() for m in modals): + logger.info("Modal/dropdown detected after click.") + return + logger.warning("Modal/dropdown did not open. Retrying click...") + + self._save_screenshot("download_btn_click_failed.png") + raise RuntimeError("Failed to open export modal after multiple clicks.") + + # ── Handle the export modal ─────────────────────────────────────────────── + + def _handle_modal(self, fields_to_select: list[str] | None) -> None: + logger.info("Waiting for CSV export modal …") + self._dismiss_upgrade_modal() + + # Find the visible modal that is actually the export modal + time.sleep(2) # Give time for the modal to open + modals = self.driver.find_elements(By.CSS_SELECTOR, ".ant-modal-content") + modal = None + for m in modals: + if m.is_displayed(): + text = m.text.lower() + if any(kw in text for kw in ("export", "field", "select", "column", "include")): + modal = m + break + + if modal is None: + # Fallback to waiting for the first visible modal + modal = WebDriverWait(self.driver, _ELEMENT_TIMEOUT).until( + EC.visibility_of_element_located((By.CSS_SELECTOR, ".ant-modal-content")) + ) + + time.sleep(1.0) # let the virtual list fully render + + # --- Optionally override field selection --- + if fields_to_select is not None: + # 1. Clear all + try: + clear_btn = modal.find_element( + By.XPATH, + ".//button[.//*[contains(text(),'Clear All')]]", + ) + self._click(clear_btn) + time.sleep(0.6) + logger.info("Cleared all field selections.") + except NoSuchElementException: + logger.warning("'Clear All' button not found — skipping.") + + # 2. Scroll through the virtual list and check each desired field + for field_val in fields_to_select: + self._select_field_in_modal(modal, field_val) + + # --- Click Export CSV --- + export_btn = None + + # 1. Look for any button with "export" or "csv" in text anywhere inside the modal + try: + buttons = modal.find_elements(By.TAG_NAME, "button") + for btn in buttons: + text = (btn.text or "").lower() + if "export" in text or "csv" in text: + export_btn = btn + break + except Exception: + pass + + # 2. Fallback to primary button inside the modal + if export_btn is None: + try: + export_btn = modal.find_element(By.CSS_SELECTOR, ".ant-btn-primary") + except NoSuchElementException: + pass + + if export_btn is None: + self._save_screenshot("export_btn_not_found.png") + raise RuntimeError("Could not find the 'Export CSV' button in modal.") + + self._click(export_btn) + logger.info("'Export CSV' button clicked.") + time.sleep(1.5) + self._dismiss_upgrade_modal() + + # def _select_field_in_modal(self, modal, field_val: str) -> None: + # """Scroll inside the modal virtual list until the checkbox is found, then check it.""" + # scrollable = modal.find_element(By.CSS_SELECTOR, ".rc-virtual-list-holder") + # max_scrolls = 20 + + # for _ in range(max_scrolls): + # try: + # # Find the label or span containing the exact field name + # label_el = modal.find_element( + # By.XPATH, + # f".//span[text()='{field_val}'] | .//label[contains(.,'{field_val}')] | .//*[text()='{field_val}']" + # ) + + # # Scroll it into view inside the virtual list container + # self.driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", label_el) + # time.sleep(0.1) + + # # Locate the parent/wrapper label and the checkbox input + # parent_label = label_el.find_element(By.XPATH, "./ancestor-or-self::label[contains(@class,'ant-checkbox-wrapper')]") + # checkbox = parent_label.find_element(By.CSS_SELECTOR, "input[type='checkbox']") + + # if not checkbox.is_selected(): + # self.driver.execute_script("arguments[0].click();", checkbox) + # time.sleep(0.2) + # logger.info(f"Selected field: {field_val}") + # return + # except (NoSuchElementException, StaleElementReferenceException): + # # Scroll down inside the modal list to reveal more items + # self.driver.execute_script( + # "arguments[0].scrollTop += 100;", scrollable + # ) + # time.sleep(1.5) + + # logger.warning(f"Field not found in modal: {field_val}") + + def _select_field_in_modal(self, modal, field_val: str) -> None: + """ + Select a field inside the Export CSV modal. + + Uses the modal's built-in search box instead of scrolling the + virtual list. This is much more reliable with Ant Design's + rc-virtual-list because only matching rows are rendered. + """ + + wait = WebDriverWait(self.driver, 15) + + try: + # Locate search input + search_box = wait.until( + lambda d: modal.find_element( + By.CSS_SELECTOR, + "input[placeholder='Search properties']" + ) + ) + + # Clear any previous search + search_box.click() + search_box.send_keys(Keys.CONTROL, "a") + search_box.send_keys(Keys.DELETE) + + # Type desired field + search_box.send_keys(field_val) + + # Wait until matching checkbox appears + checkbox = wait.until( + lambda d: modal.find_element( + By.XPATH, + f""" + .//label[ + .//*[normalize-space(text())="{field_val}"] + ]//input[@type='checkbox'] + """ + ) + ) + + # Tick if needed + if not checkbox.is_selected(): + self.driver.execute_script( + "arguments[0].click();", + checkbox + ) + + wait.until(lambda d: checkbox.is_selected()) + + logger.info(f"Selected field: {field_val}") + + except TimeoutException: + logger.warning(f"Field not found in modal: {field_val}") + + finally: + # Clear search so next iteration starts fresh + try: + search_box = modal.find_element( + By.CSS_SELECTOR, + "input[placeholder='Search properties']" + ) + + search_box.send_keys(Keys.CONTROL, "a") + search_box.send_keys(Keys.DELETE) + + wait.until( + lambda d: search_box.get_attribute("value") == "" + ) + + except Exception: + pass + + # ── Wait for the CSV download to finish ─────────────────────────────────── + + def _wait_for_download(self, timeout: int = _DOWNLOAD_TIMEOUT) -> str: + logger.info(f"Waiting up to {timeout}s for CSV download …") + deadline = time.time() + timeout + + while time.time() < deadline: + self._dismiss_upgrade_modal() + + all_csvs = glob.glob(os.path.join(self.download_dir, "*.csv")) + in_flight = glob.glob(os.path.join(self.download_dir, "*.crdownload")) + + logger.info(f"Checking downloads. CSVs: {len(all_csvs)}, in-flight: {len(in_flight)}") + + complete = [f for f in all_csvs if not f.endswith(".crdownload")] + if complete and not in_flight: + latest = max(complete, key=os.path.getmtime) + logger.info(f"Download complete → {latest}") + return latest + + time.sleep(1) + + self._save_screenshot("download_timeout.png") + raise TimeoutError( + f"CSV download did not complete within {timeout} seconds." + ) + + # ── Utilities ───────────────────────────────────────────────────────────── + + def _save_screenshot(self, filename: str) -> None: + if self.driver: + path = os.path.join(self.download_dir, filename) + try: + self.driver.save_screenshot(path) + logger.info(f"Screenshot saved → {path}") + except Exception: + pass + + def _cleanup_download_dir(self) -> None: + """Remove any leftover CSVs from previous runs.""" + for f in glob.glob(os.path.join(self.download_dir, "*.csv")): + try: + os.remove(f) + except OSError: + pass + + # ── Project switching and Modals ────────────────────────────────────────── + + def _dismiss_upgrade_modal(self) -> None: + """Find and dismiss the 'Upgrade to experience the full power of Factors' / 'Go beyond signals' modal if present.""" + try: + modals = self.driver.find_elements( + By.XPATH, + "//div[contains(@class, 'ant-modal-content')][.//*[contains(text(), 'Upgrade to experience') or contains(text(), 'Go beyond signals') or contains(text(), 'Book a Demo')]]" + ) + for m in modals: + if m.is_displayed(): + logger.info("Upgrade/Paywall modal detected! Dismissing...") + try: + close_btn = m.find_element( + By.CSS_SELECTOR, + "button.ant-modal-close, button[aria-label='Close'], .ant-modal-close-x, button[class*='close']" + ) + self._click(close_btn) + logger.info("Successfully dismissed upgrade modal.") + time.sleep(1) + except Exception as click_err: + logger.warning(f"Could not click close button on upgrade modal: {click_err}") + except Exception as e: + logger.debug(f"Error checking/dismissing upgrade modal: {e}") + + def _dismiss_popups(self) -> None: + """Attempt to dismiss any promotional or feedback modals that might block interaction.""" + self._dismiss_upgrade_modal() + logger.info("Checking for any blocking popups/modals to dismiss...") + + # 1. Look for G2 review popup / "Don't show this again" buttons + try: + g2_buttons = self.driver.find_elements( + By.XPATH, + "//button[contains(translate(text(), 'DONT SHOW', 'dont show'), 'dont show') or contains(., \"Don't show this again\")]" + ) + for btn in g2_buttons: + if btn.is_displayed(): + logger.info("Dismissing G2 / 'Don't show this again' popup.") + self._click(btn) + time.sleep(1) + except Exception as e: + logger.debug(f"Error checking/dismissing G2 popup: {e}") + + # 2. Look for general modal close buttons (e.g. ant-modal-close, ant-modal-close-x) + try: + close_buttons = self.driver.find_elements( + By.CSS_SELECTOR, + ".ant-modal-close, button.ant-modal-close, .ant-modal-close-x, [class*='ant-modal-close']" + ) + for btn in close_buttons: + if btn.is_displayed(): + # We do NOT want to close the project switch modal or fields modal! + parent_modal_text = "" + try: + parent_modal = btn.find_element(By.XPATH, "./ancestor::div[contains(@class, 'ant-modal-content')]") + parent_modal_text = parent_modal.text.lower() + except Exception: + pass + + if "switch" in parent_modal_text or "export" in parent_modal_text or "csv" in parent_modal_text: + logger.info("Skipping modal close for functional modal (switch project/export).") + continue + + logger.info("Clicking modal close button to dismiss popup.") + self._click(btn) + time.sleep(1) + except Exception as e: + logger.debug(f"Error dismissing modal popups: {e}") + + def _switch_project(self, target_project: str) -> None: + """Switch the current project to the target_project if not already active.""" + if not target_project: + logger.info("No target project specified; skipping switch.") + return + + # Dismiss any initially shown modals/popups + self._dismiss_popups() + + def clean_name(name: str) -> str: + return name.replace(" ", "").replace("-", "").replace("_", "").lower().strip() + + clean_target = clean_name(target_project) + + logger.info(f"Ensuring target project is set to: '{target_project}'") + + # 1. Locate the main project dropdown button + try: + dropdown_btn = self._wait( + By.ID, "fa-at-dropdown--projects", timeout=20, condition="clickable" + ) + except TimeoutException: + self._save_screenshot("project_dropdown_not_found.png") + raise RuntimeError("Could not find the project dropdown button (#fa-at-dropdown--projects).") + + current_project_text = dropdown_btn.text + logger.info(f"Main project button text: {current_project_text!r}") + + # If it's already active on the top nav, we're done + if clean_target in clean_name(current_project_text): + logger.info(f"Project '{target_project}' is already active.") + return + + # 2. Click the dropdown button to open the project selector + logger.info("Clicking project dropdown button...") + self._click(dropdown_btn) + time.sleep(2) + + # 3. Check the active project inside the popover/modal + try: + active_proj_div = self._wait( + By.CSS_SELECTOR, + "div[class*='ProjectModal__active_project_div']", + timeout=10, + condition="clickable" + ) + except TimeoutException: + active_proj_div = None + + if active_proj_div: + active_proj_text = active_proj_div.text + logger.info(f"Active project in modal/dropdown menu: {active_proj_text!r}") + + if clean_target in clean_name(active_proj_text): + logger.info(f"Project '{target_project}' is already selected in the active div.") + # Close the popover by pressing escape + self.driver.find_element(By.TAG_NAME, "body").send_keys(Keys.ESCAPE) + time.sleep(1) + return + + # Click active project div to reveal the project list + logger.info("Clicking active project div to open the list of all projects...") + self._click(active_proj_div) + time.sleep(2) + + # 4. Search/filter using the search input + try: + search_input = self.driver.find_element(By.CSS_SELECTOR, "input[placeholder='Search Project']") + search_input.clear() + search_input.send_keys(target_project) + logger.info(f"Typed '{target_project}' into project search input.") + time.sleep(1.5) + except NoSuchElementException: + logger.info("Project list search input not found; scanning list directly.") + + # Find the project item that matches the target project + project_items = self.driver.find_elements( + By.CSS_SELECTOR, "div[class*='ProjectModal__project_item']" + ) + target_item = None + for item in project_items: + item_text = item.text + if clean_target in clean_name(item_text): + target_item = item + break + + if not target_item: + self._save_screenshot("project_item_not_found.png") + raise RuntimeError( + f"Could not find project '{target_project}' in the project list. " + "Screenshot saved → tmp_download/project_item_not_found.png" + ) + + logger.info(f"Selecting project item: {target_item.text!r}") + self._click(target_item) + time.sleep(2) + + # 5. Handle the project switch confirmation modal if it appears + try: + switch_modal = WebDriverWait(self.driver, 5).until( + EC.visibility_of_element_located(( + By.XPATH, + "//div[contains(@class, 'ant-modal-content')][.//h4[contains(translate(text(), 'SWITCH', 'switch'), 'switch')]]" + )) + ) + logger.info("Switch confirmation modal appeared.") + switch_btn = switch_modal.find_element( + By.XPATH, + ".//button[contains(@class, 'ant-btn-primary') or .//span[text()='Switch']]" + ) + self._click(switch_btn) + logger.info("Clicked 'Switch' on confirmation modal.") + time.sleep(5) # Wait for page to reload/redirect + except TimeoutException: + logger.info("No switch confirmation modal appeared; switch may have been instant.") + + # Dismiss any popups after switching + self._dismiss_popups() + + # ── Public entry point ──────────────────────────────────────────────────── + + def run( + self, + email: str, + password: str, + target_project: str = "Inbound-Tech4Biz", + fields_to_select: list[str] | None = None, + ) -> str: + """ + Full end-to-end automation run. + + Returns: + Path to the freshly downloaded CSV file. + """ + self._cleanup_download_dir() + + try: + self._setup_driver() + self.login(email, password) + self._switch_project(target_project) + self._navigate_to_accounts() + self._click_download_button() + self._handle_modal(fields_to_select) + return self._wait_for_download() + + except Exception as exc: + self._save_screenshot("scraper_failure.png") + logger.error(f"Scraper failed: {exc}", exc_info=True) + raise + + finally: + if self.driver: + self.driver.quit() + logger.info("Browser closed.")