From ec76ef64c8dc1d5b367610a7fc767b6541870640 Mon Sep 17 00:00:00 2001 From: kenilkb Date: Wed, 1 Jul 2026 18:00:54 +0530 Subject: [PATCH] Initial --- .env.example | 8 + .gitignore | 9 + ReadMe.md | 482 +++++++++++++++++++++++++++ comparator.py | 73 +++++ config.py | 78 +++++ mailer.py | 711 ++++++++++++++++++++++++++++++++++++++++ main.py | 136 ++++++++ requirements.txt | 4 + scraper.py | 833 +++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 2334 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 ReadMe.md create mode 100644 comparator.py create mode 100644 config.py create mode 100644 mailer.py create mode 100644 main.py create mode 100644 requirements.txt create mode 100644 scraper.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..75f4e5d --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +FACTORS_EMAIL=natesh.krishnan@yourFirm.io +FACTORS_PASSWORD=Password@123 +SMTP_HOST=mail.tech4biz.org +SMTP_PORT=587 +SENDER_EMAIL=noreply@tech4biz.org +SENDER_PASSWORD=Password@123 +RECIPIENT_EMAILS=kenil@tech4biz.org, contact@tech4biz.io +HEADLESS=true \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb38501 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +venv +__pycache__ +.env +test_navigation.py +tmp_download +scratch +data +*.log +*.zip diff --git a/ReadMe.md b/ReadMe.md new file mode 100644 index 0000000..99ee88c --- /dev/null +++ b/ReadMe.md @@ -0,0 +1,482 @@ +# Factors.AI — Account Export Automation +### Automatic daily email alerts when new companies visit your website + +--- + +## What does this tool do? + +Every time you run it, this tool does **four things automatically** — no clicking, no manual work: + +``` +1. LOGS IN → Opens Factors.AI in the background and signs in with your credentials +2. DOWNLOADS → Goes to Account Profiles, clicks the export button, downloads the CSV +3. COMPARES → Checks: are there any new company domains that weren't in the last export? +4. EMAILS → If yes, sends a formatted HTML email to everyone on your list + If no new entries → does nothing (no email sent) +``` + +After every run, the fresh CSV is saved as `data/last_export.csv` — that becomes the baseline for the next comparison. + +--- + +## What the email looks like + +**Subject line examples:** + +| Situation | Subject | +|---|---| +| Very first run ever | `[Factors.AI] First Run — 483 Accounts Exported (2026-07-01)` | +| New companies found | `[Factors.AI] 7 New Accounts Detected (2026-07-01)` | +| Nothing new | *(no email sent at all)* | + +**Email body:** A styled HTML table with one row per new company, showing: + +| Account Domain | Company Name | Company Industry | Company Employee Range | Company Annual Revenue | Last Activity | +|---|---|---|---|---|---| +| dell.com | Dell | Internet Software & Services | 100K+ | $102,300,000,000 | 2026-07-01 04:48:32 | +| adobe.com | Adobe | Internet Software & Services | 10K-50K | $19,409,000,000 | 2026-06-18 11:29:45 | + +--- + +## How "new entry" is decided + +The tool uses **Account Domain** (the website address, e.g. `dell.com`) as the unique identifier for each company. + +- A company is **new** if its domain appears in today's export but was **not** in `data/last_export.csv` +- The comparison is case-insensitive (`Dell.com` = `dell.com`) +- Only `data/last_export.csv` is kept — no growing archive, no extra storage + +--- + +## Files in this folder + +``` +factors_automation/ +│ +├── main.py ← The only file you ever run ("python main.py") +├── scraper.py ← Controls the Chrome browser (login, navigate, download) +├── comparator.py ← Compares today's CSV with last_export.csv +├── mailer.py ← Builds the HTML email and sends it via SMTP +├── config.py ← All your settings live here (reads from .env) +│ +├── requirements.txt ← List of Python packages the tool needs +├── .env.example ← Template — copy this to .env and fill in your details +├── .env ← YOUR private settings (create this yourself — never share it) +│ +├── data/ +│ └── last_export.csv ← Auto-created after first run. This is the comparison baseline. +│ +├── tmp_download/ ← Temporary staging folder. Chrome downloads here first. +│ Cleaned automatically before each run. +│ Error screenshots are saved here if something goes wrong. +│ +└── automation.log ← Full log of every run (auto-created). Check this if anything fails. +``` + +> **Do not delete** `data/last_export.csv` — it is the memory of the tool. +> If you delete it, the next run will treat every account as new and email the full list. + +--- + +## Prerequisites — what you need before setup + +You need **three things** installed on the computer that will run this: + +### 1. Python 3.10 or newer +Check your version by opening a terminal and typing: +``` +python --version +``` +If it shows `Python 3.10.x` or higher, you're good. +If not, download from: https://www.python.org/downloads/ + +### 2. Google Chrome browser +The tool controls Chrome in the background. Download from: https://www.google.com/chrome/ + +> The matching ChromeDriver (what lets Python control Chrome) is downloaded **automatically** the first time you run the tool. You do not need to install it yourself. + +### 3. A Gmail account to send from (recommended) +Any Gmail account works. You will need to generate an **App Password** for it (explained in the setup below). You cannot use your regular Gmail password. + +--- + +## Setup — step by step + +### Step 1 — Download and unzip the project + +Unzip `factors_automation.zip` somewhere on your computer, for example: +- **Mac/Linux:** `~/factors_automation/` +- **Windows:** `C:\factors_automation\` + +### Step 2 — Open a terminal in that folder + +- **Mac:** Right-click the folder → "New Terminal at Folder" +- **Windows:** Open the folder in File Explorer → click the address bar → type `cmd` → press Enter + +### Step 3 — Create a virtual environment + +A virtual environment keeps this tool's packages separate from everything else on your computer. + +```bash +python -m venv venv +``` + +Then activate it: + +```bash +# Mac / Linux: +source venv/bin/activate + +# Windows: +venv\Scripts\activate +``` + +You will see `(venv)` appear at the start of your terminal prompt. This means it is active. + +### Step 4 — Install the required packages + +```bash +pip install -r requirements.txt +``` + +This downloads and installs four packages: +- **selenium** — controls the Chrome browser +- **webdriver-manager** — automatically downloads the right ChromeDriver +- **pandas** — reads and compares CSV files +- **python-dotenv** — reads your settings from the `.env` file + +### Step 5 — Create your `.env` settings file + +Copy the template: + +```bash +# Mac / Linux: +cp .env.example .env + +# Windows: +copy .env.example .env +``` + +Now open the `.env` file in any text editor (Notepad, TextEdit, VS Code, etc.) and fill in your real values. Here is what each line means: + +``` +FACTORS_EMAIL=natesh.krishnan@yourcompany.io +``` +The email address used to log in to app.factors.ai. + +``` +FACTORS_PASSWORD=YourActualPassword +``` +The password for that Factors.AI account. + +``` +SMTP_HOST=smtp.gmail.com +``` +Leave this as-is if you are using Gmail. Change only if you use a different email provider (Outlook = `smtp.office365.com`, etc.). + +``` +SMTP_PORT=587 +``` +Leave this as-is. Port 587 is the standard secure email port (STARTTLS). Works with all major email providers. + +``` +SENDER_EMAIL=your_sender@gmail.com +``` +The Gmail address the alerts will be sent **from**. + +``` +SENDER_PASSWORD=xxxx xxxx xxxx xxxx +``` +**This is NOT your regular Gmail password.** You must generate an **App Password** — see the section below. + +``` +RECIPIENT_EMAILS=person1@company.com,person2@company.com +``` +Everyone who should receive the alert emails. Separate multiple addresses with commas. No spaces around the commas. + +``` +HEADLESS=true +``` +`true` means Chrome runs invisibly in the background (recommended for scheduled/daily runs). +`false` means you can watch Chrome open and do its work on screen (useful when testing or debugging). + +--- + +### How to generate a Gmail App Password + +Gmail does not allow scripts to log in with your regular password. You must create a special one-time App Password. + +1. Go to your Google Account: https://myaccount.google.com/ +2. Click **Security** in the left sidebar +3. Under "How you sign in to Google", click **2-Step Verification** and enable it if not already on +4. Go back to Security → scroll down → click **App Passwords** +5. Under "Select app" choose **Mail** → under "Select device" choose **Other** → type `FactorsAI Bot` +6. Click **Generate** +7. Google shows a 16-character code like `abcd efgh ijkl mnop` +8. Copy it (including spaces) into your `.env` file as `SENDER_PASSWORD` + +> This App Password only lets the script send emails. It cannot access your Gmail inbox or change your account. + +--- + +## Choosing which columns to export + +By default (`FIELDS_TO_SELECT = None` in `config.py`), the tool keeps whichever columns are **already checked** in the Factors.AI export modal — which is the default 5: + +- Company Name +- Company Industry +- Company Employee Range +- Company Annual Revenue +- Last Activity + +The downloaded CSV will also always include **Account Domain** as the first column (this is always exported by Factors.AI and is used as the unique comparison key). + +### To change which columns are exported + +Open `config.py` in a text editor. Find this section near the bottom: + +```python +FIELDS_TO_SELECT: list[str] | None = None # None = use defaults +``` + +Change it to a list of the fields you want. For example, to export only Company Name, Industry, and Last Activity: + +```python +FIELDS_TO_SELECT = [ + "$6Signal_name", + "$6Signal_industry", + "last_activity", +] +``` + +**Full list of available field codes:** + +| Code | Column name in CSV | +|---|---| +| `"$6Signal_name"` | Company Name | +| `"$6Signal_industry"` | Company Industry | +| `"$6Signal_employee_range"` | Company Employee Range | +| `"$6Signal_annual_revenue"` | Company Annual Revenue | +| `"last_activity"` | Last Activity | +| `"$tag_hidden"` | Tags Hidden | +| `"$latest_source"` | Account Latest Source | +| `"$latest_campaign"` | Account Latest Campaign | +| `"$initial_campaign"` | Account First Campaign | +| `"$account_activity_url"` | Account Activity URL | +| `"$domain_name"` | Company ID | +| `"$latest_page_url"` | Account Latest Page URL | + +When you set `FIELDS_TO_SELECT` to a list, the tool first clicks "Clear All" in the modal, then checks exactly the fields you listed. Setting it back to `None` restores the default behaviour. + +--- + +## Running the tool + +Make sure `(venv)` is active in your terminal before running. + +### Normal run (invisible browser, sends real email) +```bash +python main.py +``` + +### Watch mode — opens real Chrome window (good for testing) +```bash +python main.py --visible +``` +Use this the first time you run it to verify everything works correctly. + +### Dry run — downloads and compares, but does NOT send any email +```bash +python main.py --dry-run +``` +The new entries are printed to the terminal instead. Useful for checking what would have been emailed. + +### Combine both flags +```bash +python main.py --visible --dry-run +``` + +--- + +## What happens during a run (step by step) + +When you run `python main.py`, here is exactly what happens internally: + +**Step 1 — Download CSV** +- Chrome launches (invisibly unless `--visible`) +- Opens `https://app.factors.ai/` +- Enters email and password from your `.env` +- Waits up to 30 seconds for login to complete +- Finds and clicks the Account Profiles page in the sidebar navigation. If that fails, it tries five known URL patterns (`/accounts`, `/account-profiles`, `/accounts/profiles`, `/analytics/accounts`, `/v2/accounts`) +- Clicks the download (↓) toolbar icon +- The export modal appears — columns are selected (or kept as default) +- Clicks "Export CSV" +- Waits up to 90 seconds for the file to appear in `tmp_download/` +- Chrome closes + +**Step 2 — Compare** +- Loads `tmp_download/.csv` and `data/last_export.csv` +- Compares every row's **Account Domain** (first column) +- Identifies domains in the new file that were not in the old file +- If `data/last_export.csv` does not exist → all rows are treated as new (first run) + +**Step 3 — Email** *(skipped if zero new entries and not first run)* +- Builds an HTML email with a table of new rows +- Connects to `smtp.gmail.com:587` via STARTTLS +- Logs in with `SENDER_EMAIL` and `SENDER_PASSWORD` +- Sends the email to all `RECIPIENT_EMAILS` + +**Step 4 — Save** +- Copies the downloaded CSV to `data/last_export.csv`, replacing the previous one +- This becomes the new baseline for the next run + +Everything is logged to `automation.log` in real time. + +--- + +## Scheduling — run automatically every day + +### Mac / Linux (using cron) + +Open the cron editor: +```bash +crontab -e +``` + +This runs every day at 01:30 UTC, which is 07:00 IST. (adjust the paths to match your actual folder): +``` +30 1 * * * /full/path/to/factors_automation/venv/bin/python /full/path/to/factors_automation/main.py +``` + +To find your full path, run this command inside the project folder: +```bash +pwd +``` + +Example result: `/home/natesh/factors_automation` + +So the cron line would be: +``` +0 8 * * * /home/natesh/factors_automation/venv/bin/python /home/natesh/factors_automation/main.py +``` + +### Windows (using Task Scheduler) + +1. Open **Task Scheduler** (search for it in the Start menu) +2. Click **Create Basic Task** on the right +3. Name: `Factors.AI Export` +4. Trigger: **Daily** at your preferred time +5. Action: **Start a program** +6. Program: `C:\factors_automation\venv\Scripts\python.exe` +7. Arguments: `C:\factors_automation\main.py` +8. Click **Finish** + +--- + +## Checking if a run succeeded + +Open `automation.log` in any text editor. A successful run looks like this: + +``` +2026-07-01 08:00:01 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:01 [INFO ] __main__ — Factors.AI Account Export Automation — START +2026-07-01 08:00:01 [INFO ] __main__ — headless=True dry_run=False +2026-07-01 08:00:01 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:01 [INFO ] __main__ — STEP 1: Downloading CSV … +2026-07-01 08:00:04 [INFO ] scraper — Opening https://app.factors.ai/ … +2026-07-01 08:00:09 [INFO ] scraper — Email entered. +2026-07-01 08:00:10 [INFO ] scraper — Password entered. +2026-07-01 08:00:11 [INFO ] scraper — Login form submitted. +2026-07-01 08:00:14 [INFO ] scraper — Logged in successfully. URL: https://app.factors.ai/accounts +2026-07-01 08:00:18 [INFO ] scraper — Download button clicked. +2026-07-01 08:00:21 [INFO ] scraper — 'Export CSV' button clicked. +2026-07-01 08:00:24 [INFO ] scraper — Download complete → tmp_download/accounts.csv +2026-07-01 08:00:24 [INFO ] scraper — Browser closed. +2026-07-01 08:00:24 [INFO ] __main__ — STEP 2: Comparing with previous export … +2026-07-01 08:00:24 [INFO ] comparator — Comparison complete: 7 new / 490 total (previous had 483 rows) +2026-07-01 08:00:24 [INFO ] __main__ — ✓ New entries detected (7 new) +2026-07-01 08:00:24 [INFO ] __main__ — STEP 3: Sending alert email … +2026-07-01 08:00:26 [INFO ] mailer — Email sent to: ['you@company.com'] | subject: [Factors.AI] 7 New Accounts Detected (2026-07-01) +2026-07-01 08:00:26 [INFO ] __main__ — STEP 4: Saving export as last_export.csv +2026-07-01 08:00:26 [INFO ] __main__ — ================================================================= +2026-07-01 08:00:26 [INFO ] __main__ — Automation complete ✓ +``` + +--- + +## Troubleshooting + +### "Could not find email input on the login page" +The tool took a screenshot. Open `tmp_download/login_page_not_found.png` to see what the browser was looking at. + +Possible causes: +- Factors.AI is down or showing a maintenance page +- Your internet connection is slow (increase `_PAGE_LOAD_TIMEOUT` in `scraper.py` from `30` to `60`) +- The login page layout changed (contact whoever maintains this tool) + +### "Still on login page after submission — check credentials" +Your `FACTORS_EMAIL` or `FACTORS_PASSWORD` in `.env` is wrong. Check and correct them. +Screenshot saved as `tmp_download/post_login_timeout.png`. + +### "Could not navigate to the Account Profiles page" +The tool could not find the Accounts section. Screenshot saved as `tmp_download/accounts_not_found.png`. + +Try running with `--visible` to watch what happens: +```bash +python main.py --visible --dry-run +``` + +### "CSV download did not complete within 90 seconds" +The download took too long. Possible causes: +- Very large export (many thousands of rows) +- Slow internet + +Open `scraper.py`, find the line `_DOWNLOAD_TIMEOUT = 90` near the top, and change `90` to `180`. + +### Gmail error: "Username and Password not accepted" +You used your regular Gmail password instead of an App Password. +Follow the **"How to generate a Gmail App Password"** steps above. + +### Gmail error: "SMTPAuthenticationError" +Same as above, or 2-Step Verification is not enabled on the Gmail account. + +### `SyntaxError` or `python: command not found` +Your Python version is older than 3.10. Check with `python --version` and upgrade at https://www.python.org/downloads/ + +### The `(venv)` prefix disappeared from my terminal +Your virtual environment is no longer active. Reactivate it: +```bash +# Mac / Linux: +source venv/bin/activate + +# Windows: +venv\Scripts\activate +``` + +--- + +## Security notes + +- Your `.env` file contains passwords. **Never share it, email it, or commit it to Git.** +- The `.env.example` file contains only placeholder text and is safe to share. +- The Gmail App Password only allows the script to send email. It cannot read your inbox, delete messages, or change your Google account settings. +- `data/last_export.csv` contains company names and domains from Factors.AI. Treat it with the same care as any customer data. + +--- + +## Quick-reference card + +| Task | Command | +|---|---| +| Normal run (background, sends email) | `python main.py` | +| Watch browser work on screen | `python main.py --visible` | +| Test without sending email | `python main.py --dry-run` | +| Watch + test, no email | `python main.py --visible --dry-run` | + +| File | Purpose | +|---|---| +| `.env` | Your private credentials and settings | +| `config.py` | Export field selection and advanced settings | +| `data/last_export.csv` | Comparison baseline — do not delete | +| `automation.log` | Full run history — check when something fails | +| `tmp_download/*.png` | Error screenshots — check when a step fails | \ No newline at end of file diff --git a/comparator.py b/comparator.py new file mode 100644 index 0000000..06c3dc7 --- /dev/null +++ b/comparator.py @@ -0,0 +1,73 @@ +""" +comparator.py — Compares two Account Profiles CSVs and returns new entries. + +Logic +───── +• Unique key : "Account Domain" (the website domain, e.g. dell.com) +• New entry : a domain present in the NEW csv that is absent in the OLD csv. +• If OLD csv is missing (first ever run) → every row counts as "new". +""" +import logging +from pathlib import Path + +import pandas as pd + +logger = logging.getLogger(__name__) + +# Column used to identify a unique account across exports +_UNIQUE_KEY = "Account Domain" + + +def _load(path: str) -> pd.DataFrame: + return pd.read_csv(path, dtype=str).fillna("-").rename( + columns=lambda c: c.strip() + ) + + +def find_new_entries( + new_csv: str, + old_csv: str | None, +) -> tuple[pd.DataFrame, bool]: + """ + Parameters + ---------- + new_csv : path to the freshly downloaded CSV. + old_csv : path to the previously stored CSV, or None. + + Returns + ------- + (new_entries_df, is_first_run) + new_entries_df — DataFrame of rows that are brand-new. + is_first_run — True when there was no previous CSV to compare against. + """ + new_df = _load(new_csv) + + if _UNIQUE_KEY not in new_df.columns: + # The downloaded CSV might have a slightly different column order; + # treat the first column as the domain key. + logger.warning( + f"Column '{_UNIQUE_KEY}' not found. " + f"Available columns: {list(new_df.columns)}" + ) + first_col = new_df.columns[0] + new_df.rename(columns={first_col: _UNIQUE_KEY}, inplace=True) + logger.warning(f"Using '{first_col}' as the unique key instead.") + + if old_csv is None or not Path(old_csv).exists(): + logger.info("No previous CSV found — treating ALL entries as new (first run).") + return new_df, True + + old_df = _load(old_csv) + old_keys = set( + old_df[_UNIQUE_KEY].str.strip().str.lower() + if _UNIQUE_KEY in old_df.columns + else old_df.iloc[:, 0].str.strip().str.lower() + ) + mask = ~new_df[_UNIQUE_KEY].str.strip().str.lower().isin(old_keys) + new_entries = new_df[mask].copy().reset_index(drop=True) + + logger.info( + f"Comparison complete: {len(new_entries)} new / " + f"{len(new_df)} total (previous had {len(old_df)} rows)" + ) + return new_entries, False diff --git a/config.py b/config.py new file mode 100644 index 0000000..09227dc --- /dev/null +++ b/config.py @@ -0,0 +1,78 @@ +""" +config.py — Central configuration for Factors.AI automation. +All secrets are loaded from .env (copy .env.example → .env and fill in values). +""" +from argparse import RawDescriptionHelpFormatter +import os +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + +# ── Paths ───────────────────────────────────────────────────────────────────── +BASE_DIR = Path(__file__).resolve().parent +DATA_DIR = BASE_DIR / "data" +TEMP_DOWNLOAD_DIR = BASE_DIR / "tmp_download" +LAST_CSV_PATH = DATA_DIR / "last_export.csv" +LOG_PATH = BASE_DIR / "automation.log" + +DATA_DIR.mkdir(parents=True, exist_ok=True) +TEMP_DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) + +# ── Factors.AI credentials ──────────────────────────────────────────────────── +FACTORS_EMAIL = os.getenv("FACTORS_EMAIL", "Add_your_mail@env.com") +FACTORS_PASSWORD = os.getenv("FACTORS_PASSWORD", "Admin@123") +TARGET_PROJECT = os.getenv("TARGET_PROJECT", "Inbound-Tech4Biz") + +# ── Email / SMTP settings ───────────────────────────────────────────────────── +# For Gmail: enable 2FA, create an App Password and use it as SENDER_PASSWORD. +SMTP_HOST = os.getenv("SMTP_HOST", "smtp.zoho.com") +SMTP_PORT = int(os.getenv("SMTP_PORT", "587")) +SENDER_EMAIL = os.getenv("SENDER_EMAIL", "your_sender@gmail.com") +SENDER_PASSWORD = os.getenv("SENDER_PASSWORD", "your_app_password") + +# Comma-separated list of recipient emails +RECIPIENT_EMAILS: list[str] = [ + e.strip() + for e in os.getenv("RECIPIENT_EMAILS", "recipient@example.com").split(",") + if e.strip() +] + +# ── Fields to select in the CSV export modal ────────────────────────────────── +# Set to None → keep whatever boxes are pre-checked (default 5 fields). +# Set to list → clear all, then check exactly these field values. +# +# Known field values (from the modal checkboxes): +# "$6Signal_name" Company Name +# "$6Signal_industry" Company Industry +# "$6Signal_employee_range" Company Employee Range +# "$6Signal_annual_revenue" Company Annual Revenue +# "last_activity" Last Activity +# "$tag_hidden" Tags Hidden +# "$latest_source" Account Latest Source +# "$latest_campaign" Account Latest Campaign +# "$initial_campaign" Account First Campaign +# "$account_activity_url" Account Activity URL +# "$domain_name" Company ID +# "$latest_page_url" Account Latest Page URL +# +# FIELDS_TO_SELECT: list[str] | None = None # None = use defaults + +# Uncomment ↓ to specify exact fields instead of defaults: +FIELDS_TO_SELECT = [ +"Company Name", # "$6Signal_name", +"Company Industry", # "$6Signal_industry", +"Company Employee Range", # "$6Signal_employee_range", +"Company Annual Revenue", # "$6Signal_annual_revenue", +"Last Activity", # "last_activity", +"Company LinkedIn URL", # "$enriched_company_linkedin_url", +"Company Description", # "$enriched_company_description", +"Company Domain", # "$6Signal_domain", +"Company HQ Address", # "$6Signal_address", +"Company HQ City", # "$6Signal_city", +"Company Employee Count" # "$6Signal_employee_count", +] + +# ── Browser settings ────────────────────────────────────────────────────────── +# HEADLESS=false opens a real browser window (handy for debugging). +HEADLESS = os.getenv("HEADLESS", "true").lower() == "true" diff --git a/mailer.py b/mailer.py new file mode 100644 index 0000000..0c4d446 --- /dev/null +++ b/mailer.py @@ -0,0 +1,711 @@ +""" +mailer.py — Sends a polished HTML digest of new Factors.AI account entries, +with the full export attached as a CSV file. + +Design notes +------------ +* Card-per-account layout instead of a single wide table. An 11-column table + cannot fit inside an email viewport on any client, and Outlook desktop + (Word rendering engine) cannot horizontally scroll — a wide table there + just renders broken/clipped. Stacked cards render correctly everywhere, + including Outlook, Gmail, Apple Mail and mobile clients, with zero + horizontal overflow. +* All styling is inline (email clients strip most + + + + + + + + +
+ + + + + + + + + + + + + + {f'" if False else ""} + {attachment_html} + + + + + + + + + + + + +
+ +""" + + +def _build_plain_text(df: pd.DataFrame, is_first_run: bool, csv_filename: Optional[str]) -> str: + count = len(df) + lines = ["Factors.AI - Account Alerts", "=" * 32, ""] + if is_first_run: + lines.append(f"Baseline export - {count} account(s) currently in Factors.AI.") + else: + lines.append(f"{count} new account(s) detected since the last export.") + lines.append("") + + for _, row in df.iterrows(): + name = _clean(row.get(_PRIMARY_FIELD)) or _clean(row.get(_DOMAIN_FIELD)) or "Unknown company" + domain = _clean(row.get(_DOMAIN_FIELD)) + header = name + (f" ({domain})" if domain and domain != name else "") + lines.append(f"- {header}") + for field in [*_BADGE_FIELDS, *_GRID_FIELDS, _ADDRESS_FIELD]: + val = _clean(row.get(field)) + if val: + lines.append(f" {_display_label(field)}: {val}") + linkedin = _clean(row.get(_LINKEDIN_FIELD)) + if linkedin: + lines.append(f" LinkedIn: {linkedin}") + lines.append("") + + if csv_filename: + lines.append(f"Full export attached: {csv_filename}") + lines.append("") + lines.append("This is an automated message from the Factors.AI Account Export script.") + return "\n".join(lines) + + +# ── Attachment ──────────────────────────────────────────────────────────────── + +def _attach_csv(msg: MIMEMultipart, csv_path: Optional[str]) -> Optional[str]: + """Attach the exported CSV to the message. Returns the filename attached, or None.""" + if not csv_path: + logger.info("No downloaded_csv path provided — sending without an attachment.") + return None + + path = Path(csv_path) + if not path.is_file(): + logger.warning(f"downloaded_csv not found, skipping attachment: {path}") + return None + + try: + data = path.read_bytes() + except OSError as exc: + logger.warning(f"Could not read downloaded_csv ({path}): {exc} — skipping attachment.") + return None + + part = MIMEApplication(data, _subtype="csv") + part.add_header("Content-Disposition", "attachment", filename=path.name) + msg.attach(part) + logger.info(f"Attached CSV: {path.name} ({len(data):,} bytes)") + return path.name + + +# ── Public API ──────────────────────────────────────────────────────────────── + +def send_alert( + smtp_host: str, + smtp_port: int, + sender_email: str, + sender_password: str, + recipients: Sequence[str], + new_entries: pd.DataFrame, + downloaded_csv: Optional[str] = None, + *, + is_first_run: bool = False, + skip_if_empty: bool = True, + use_ssl: bool = False, +) -> bool: + """ + Compose and send the Factors.AI account alert email, with the raw CSV export attached. + + Parameters + ---------- + smtp_host / smtp_port : SMTP server. Use use_ssl=True for port 465 (implicit TLS), + or leave use_ssl=False for STARTTLS (typically port 587). + sender_email : The From address. + sender_password : SMTP password / App Password. + recipients : List of To addresses. Must be non-empty. + new_entries : DataFrame of new rows to include in the digest. + downloaded_csv : Path to the full CSV export to attach. If missing/unreadable, + the email still sends without the attachment (logged as a warning). + is_first_run : True if no previous CSV existed (baseline export copy/subject). + skip_if_empty : If True (default) and new_entries is empty and this is not the + first run, no email is sent — avoids empty "0 new accounts" noise. + use_ssl : Use SMTP_SSL instead of STARTTLS. + + Returns + ------- + bool — True if an email was sent, False if sending was skipped (empty digest). + """ + if not recipients: + raise ValueError("recipients must contain at least one address") + if new_entries is None: + raise ValueError("new_entries cannot be None") + + count = len(new_entries) + if count == 0 and not is_first_run and skip_if_empty: + logger.info("No new accounts detected — skipping email (skip_if_empty=True).") + return False + + today = datetime.now().strftime("%Y-%m-%d") + subject = ( + f"[Factors.AI] Baseline Export — {count} Account{'s' if count != 1 else ''} ({today})" + if is_first_run + else f"[Factors.AI] {count} New Account{'s' if count != 1 else ''} Detected ({today})" + ) + + msg = MIMEMultipart("mixed") + msg["Subject"] = subject + msg["From"] = sender_email + msg["To"] = ", ".join(recipients) + msg["Date"] = formatdate(localtime=True) + msg["Message-ID"] = make_msgid() + + csv_filename = Path(downloaded_csv).name if downloaded_csv and Path(downloaded_csv).is_file() else None + + alt = MIMEMultipart("alternative") + alt.attach(MIMEText(_build_plain_text(new_entries, is_first_run, csv_filename), "plain", "utf-8")) + alt.attach(MIMEText(_build_html(new_entries, is_first_run, csv_filename), "html", "utf-8")) + msg.attach(alt) + + _attach_csv(msg, downloaded_csv) + + logger.info(f"Connecting to {smtp_host}:{smtp_port} ...") + try: + smtp_cls = smtplib.SMTP_SSL if use_ssl else smtplib.SMTP + with smtp_cls(smtp_host, smtp_port, timeout=30) as server: + server.ehlo() + if not use_ssl: + server.starttls() + server.ehlo() + server.login(sender_email, sender_password) + server.sendmail(sender_email, list(recipients), msg.as_string()) + except smtplib.SMTPException: + logger.exception(f"Failed to send Factors.AI alert email to {recipients}") + raise + + logger.info(f"Email sent to: {recipients} | subject: {subject}") + return True + + +def save_preview( + new_entries: pd.DataFrame, + output_path: str = "preview.html", + is_first_run: bool = False, + csv_filename: Optional[str] = "factors_ai_export.csv", +) -> str: + """Render the email to a local HTML file without sending anything — for quick design checks.""" + html_body = _build_html(new_entries, is_first_run, csv_filename) + Path(output_path).write_text(html_body, encoding="utf-8") + return output_path + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# """ +# mailer.py — Sends an HTML email listing new Factors.AI account entries. +# """ +# import logging +# import smtplib +# from datetime import datetime +# from email.mime.multipart import MIMEMultipart +# from email.mime.text import MIMEText + +# import pandas as pd + +# logger = logging.getLogger(__name__) + +# # Columns to show in the email table (in this order) +# _EMAIL_COLS = [ +# "Account Domain", +# "Company Name", +# "Company Industry", +# "Company Employee Range", +# "Company Annual Revenue", +# "Last Activity", +# ] + +# # ── HTML helpers ────────────────────────────────────────────────────────────── + +# _STYLES = { +# "body": "font-family:Arial,sans-serif;font-size:14px;color:#333;margin:0;padding:0;background:#f4f6f9", +# "wrapper": "max-width:900px;margin:32px auto;background:#fff;border-radius:8px;overflow:hidden;box-shadow:0 2px 8px rgba(0,0,0,.12)", +# "header": "background:#4F46E5;padding:24px 32px", +# "h1": "color:#fff;margin:0;font-size:22px", +# "sub": "color:#c7d2fe;font-size:13px;margin:6px 0 0", +# "body_div": "padding:24px 32px", +# "table": "border-collapse:collapse;width:100%;font-size:13px", +# "th": "padding:10px 14px;background:#4F46E5;color:#fff;text-align:left;white-space:nowrap", +# "td_even": "padding:9px 14px;border-bottom:1px solid #eef0f3;background:#f9fafb", +# "td_odd": "padding:9px 14px;border-bottom:1px solid #eef0f3;background:#fff", +# "footer": "padding:16px 32px;background:#f4f6f9;font-size:11px;color:#9ca3af", +# } + + +# def _make_table(df: pd.DataFrame) -> str: +# cols = [c for c in _EMAIL_COLS if c in df.columns] +# if not cols: +# cols = list(df.columns) + +# headers = "".join( +# f"{c}" for c in cols +# ) + +# rows_html = "" +# for i, (_, row) in enumerate(df[cols].iterrows()): +# td_style = _STYLES["td_even"] if i % 2 == 0 else _STYLES["td_odd"] +# cells = "".join( +# f"{str(v)}" for v in row +# ) +# rows_html += f"{cells}" + +# tbl_style = _STYLES["table"] +# return ( +# f"" +# f"{headers}" +# f"{rows_html}" +# f"
" +# ) + + +# def _build_html(df: pd.DataFrame, is_first_run: bool) -> str: +# count = len(df) +# run_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + +# if is_first_run: +# intro = ( +# "This is the first run of the automation — no previous " +# "export exists for comparison. All current accounts are listed below." +# ) +# else: +# intro = ( +# f"{count} new account{'s' if count != 1 else ''} " +# f"{'have' if count != 1 else 'has'} been detected since the last export." +# ) + +# table_html = _make_table(df) + +# return f""" +# +# +#
+ +#
+#

Factors.AI — Account Profiles Report

+#

Run time: {run_time}

+#
+ +#
+#

{intro}

+#

+# Total rows in this export: {count} +#

+#
+# {table_html} +#
+ +#
+# This is an automated message generated by the Factors.AI Account +# Export script. Do not reply to this email. +#
+ +#
+# +# +# """ + + +# # ── Public API ──────────────────────────────────────────────────────────────── + +# def send_alert( +# smtp_host: str, +# smtp_port: int, +# sender_email: str, +# sender_password: str, +# recipients: list[str], +# new_entries: pd.DataFrame, +# is_first_run: bool = False, +# downloaded_csv: str, +# ) -> None: +# """ +# Compose and send the alert email. + +# Parameters +# ---------- +# smtp_host / smtp_port : SMTP server (default: Gmail STARTTLS 587). +# sender_email : The From address. +# sender_password : SMTP password / App Password. +# recipients : List of To addresses. +# new_entries : DataFrame of new rows to include. +# is_first_run : True if no previous CSV existed. +# """ +# count = len(new_entries) +# today = datetime.now().strftime("%Y-%m-%d") + +# subject = ( +# f"[Factors.AI] First Run — {count} Accounts Exported ({today})" +# if is_first_run +# else f"[Factors.AI] {count} New Account{'s' if count != 1 else ''} Detected ({today})" +# ) + +# html_body = _build_html(new_entries, is_first_run) + +# msg = MIMEMultipart("alternative") +# msg["Subject"] = subject +# msg["From"] = sender_email +# msg["To"] = ", ".join(recipients) +# msg.attach(MIMEText(html_body, "html", "utf-8")) + +# logger.info(f"Connecting to {smtp_host}:{smtp_port} …") +# with smtplib.SMTP(smtp_host, smtp_port, timeout=30) as server: +# server.ehlo() +# server.starttls() +# server.login(sender_email, sender_password) +# server.sendmail(sender_email, recipients, msg.as_string()) + +# logger.info(f"Email sent to: {recipients} | subject: {subject}") diff --git a/main.py b/main.py new file mode 100644 index 0000000..0fe7533 --- /dev/null +++ b/main.py @@ -0,0 +1,136 @@ +""" +main.py — Entry point for the Factors.AI account-export automation. + +Usage +───── + python main.py # uses settings in config.py / .env + python main.py --visible # opens a real browser window (debug) + python main.py --dry-run # skip email; just download & compare + +Schedule (cron example — runs every day at 8 AM) +───────────────────────────────────────────────── + 0 8 * * * /path/to/venv/bin/python /path/to/factors_automation/main.py +""" +import argparse +import logging +import shutil +import sys +from pathlib import Path + +# ── Logging (set up before any other import) ────────────────────────────────── +from config import LOG_PATH # noqa: E402 – needs to be early + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)-8s] %(name)s — %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler(str(LOG_PATH), encoding="utf-8"), + ], +) +logger = logging.getLogger(__name__) + +# ── Application imports ─────────────────────────────────────────────────────── +from config import ( # noqa: E402 + FACTORS_EMAIL, + FACTORS_PASSWORD, + TARGET_PROJECT, + TEMP_DOWNLOAD_DIR, + LAST_CSV_PATH, + SMTP_HOST, + SMTP_PORT, + SENDER_EMAIL, + SENDER_PASSWORD, + RECIPIENT_EMAILS, + FIELDS_TO_SELECT, + HEADLESS, +) +from scraper import FactorsAIScraper +from comparator import find_new_entries +from mailer import send_alert + + +# ── CLI arguments ───────────────────────────────────────────────────────────── + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description="Factors.AI account-export automation") + p.add_argument( + "--visible", + action="store_true", + help="Open a real browser window instead of running headless", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Download and compare but do NOT send the email", + ) + return p.parse_args() + + +# ── Main flow ───────────────────────────────────────────────────────────────── + +def main() -> None: + args = _parse_args() + headless = HEADLESS and not args.visible + + logger.info("=" * 65) + logger.info(" Factors.AI Account Export Automation — START") + logger.info(f" headless={headless} dry_run={args.dry_run}") + logger.info("=" * 65) + + # ── Step 1 ─ Download CSV via Selenium ──────────────────────────────────── + logger.info("STEP 1: Downloading CSV …") + scraper = FactorsAIScraper( + download_dir=str(TEMP_DOWNLOAD_DIR), + headless=headless, + ) + downloaded_csv = scraper.run( + email=FACTORS_EMAIL, + password=FACTORS_PASSWORD, + target_project=TARGET_PROJECT, + fields_to_select=FIELDS_TO_SELECT, + ) + logger.info(f" Downloaded → {downloaded_csv}") + + # ── Step 2 ─ Compare with last export ──────────────────────────────────── + logger.info("STEP 2: Comparing with previous export …") + last_csv = str(LAST_CSV_PATH) if LAST_CSV_PATH.exists() else None + new_entries, is_first_run = find_new_entries(downloaded_csv, last_csv) + + if new_entries.empty and not is_first_run: + logger.info(" ✓ No new entries found — nothing to mail.") + else: + count = len(new_entries) + tag = "(first run — all entries)" if is_first_run else f"({count} new)" + logger.info(f" ✓ New entries detected {tag}") + + # ── Step 3 ─ Send email ─────────────────────────────────────────────── + if args.dry_run: + logger.info("STEP 3: [DRY RUN] Email skipped.") + logger.info(new_entries.to_string()) + else: + logger.info("STEP 3: Sending alert email …") + send_alert( + smtp_host=SMTP_HOST, + smtp_port=SMTP_PORT, + sender_email=SENDER_EMAIL, + sender_password=SENDER_PASSWORD, + recipients=RECIPIENT_EMAILS, + new_entries=new_entries, + is_first_run=is_first_run, + downloaded_csv=downloaded_csv + ) + logger.info(" ✓ Email sent.") + + # ── Step 4 ─ Persist the new CSV as last_export.csv ───────────────────── + logger.info(f"STEP 4: Saving export as last_export.csv → {LAST_CSV_PATH}") + shutil.copy2(downloaded_csv, str(LAST_CSV_PATH)) + + logger.info("=" * 65) + logger.info(" Automation complete ✓") + logger.info("=" * 65) + + +if __name__ == "__main__": + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..aad0a21 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +selenium +webdriver-manager +pandas +python-dotenv \ No newline at end of file diff --git a/scraper.py b/scraper.py new file mode 100644 index 0000000..b5ab7ca --- /dev/null +++ b/scraper.py @@ -0,0 +1,833 @@ +""" +scraper.py — Selenium automation to log in to Factors.AI and download the + Account Profiles CSV. +""" +import glob +import logging +import os +import random +import time +from pathlib import Path + +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from webdriver_manager.chrome import ChromeDriverManager +from selenium.common.exceptions import TimeoutException + +logger = logging.getLogger(__name__) + +_PAGE_LOAD_TIMEOUT = 30 # seconds to wait for page transitions +_ELEMENT_TIMEOUT = 20 # seconds to wait for elements +_DOWNLOAD_TIMEOUT = 90 # seconds to wait for file download + + +class FactorsAIScraper: + """Downloads the Account Profiles CSV export from app.factors.ai.""" + + def __init__(self, download_dir: str, headless: bool = True): + self.download_dir = str(Path(download_dir).resolve()) + self.headless = headless + self.driver: webdriver.Chrome | None = None + + # ── Driver setup ────────────────────────────────────────────────────────── + + def _setup_driver(self) -> None: + options = Options() + + if self.headless: + options.add_argument("--headless=new") + + # Use a persistent Chrome profile to keep cookies and session active + profile_dir = os.path.join(self.download_dir, "chrome_profile") + os.makedirs(profile_dir, exist_ok=True) + # Clean up stale chrome locks + lock_file = os.path.join(profile_dir, "SingletonLock") + if os.path.islink(lock_file) or os.path.exists(lock_file): + try: + os.remove(lock_file) + logger.info("Cleaned up stale SingletonLock from Chrome profile.") + except Exception as e: + logger.warning(f"Could not remove SingletonLock: {e}") + options.add_argument(f"--user-data-dir={profile_dir}") + + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-gpu") + options.add_argument("--window-size=1920,1080") + options.add_argument("--disable-extensions") + options.add_argument("--disable-blink-features=AutomationControlled") + options.add_experimental_option("excludeSwitches", ["enable-automation"]) + options.add_experimental_option("useAutomationExtension", False) + + prefs = { + "download.default_directory": self.download_dir, + "download.prompt_for_download": False, + "download.directory_upgrade": True, + "safebrowsing.enabled": False, + "safebrowsing.disable_download_protection": True, + } + options.add_experimental_option("prefs", prefs) + + service = Service(ChromeDriverManager().install()) + self.driver = webdriver.Chrome(service=service, options=options) + self.driver.set_page_load_timeout(_PAGE_LOAD_TIMEOUT) + + # Enable downloads explicitly when headless (CDP call) + if self.headless: + try: + self.driver.execute_cdp_cmd( + "Page.setDownloadBehavior", + {"behavior": "allow", "downloadPath": self.download_dir}, + ) + except Exception: + pass # Older ChromeDriver builds may not support this + + # ── Helpers ─────────────────────────────────────────────────────────────── + + def _wait( + self, + by: str, + value: str, + timeout: int = _ELEMENT_TIMEOUT, + condition: str = "presence", + ): + ec_map = { + "presence": EC.presence_of_element_located, + "clickable": EC.element_to_be_clickable, + "visible": EC.visibility_of_element_located, + } + return WebDriverWait(self.driver, timeout).until( + ec_map[condition]((by, value)) + ) + + def _try_selectors(self, selectors: list[tuple], timeout: int = 8): + """Return the first element found from a list of (By.*, selector) pairs, waiting at most timeout seconds total.""" + deadline = time.time() + timeout + while True: + for by, sel in selectors: + try: + elements = self.driver.find_elements(by, sel) + for el in elements: + if el.is_displayed() and el.is_enabled(): + return el + except Exception: + pass + if time.time() > deadline: + break + time.sleep(0.2) + return None + + def _click(self, element) -> None: + """Click an element, falling back to JS click if intercepted.""" + try: + element.click() + except Exception as e: + logger.warning(f"Standard click failed ({e}), attempting JS click...") + self.driver.execute_script("arguments[0].click();", element) + + def _type_humanlike(self, element, text: str) -> None: + """Type text into an element with a short random delay between keystrokes.""" + element.click() + from selenium.webdriver.common.keys import Keys + try: + element.send_keys(Keys.CONTROL + "a") + element.send_keys(Keys.BACKSPACE) + except Exception: + element.clear() + + for char in text: + element.send_keys(char) + time.sleep(random.uniform(0.04, 0.12)) + + try: + self.driver.execute_script("arguments[0].dispatchEvent(new Event('input', { bubbles: true }));", element) + self.driver.execute_script("arguments[0].dispatchEvent(new Event('change', { bubbles: true }));", element) + except Exception: + pass + + # ── Login ───────────────────────────────────────────────────────────────── + + def login(self, email: str, password: str) -> None: + logger.info("Opening https://app.factors.ai/ …") + self.driver.get("https://app.factors.ai/") + time.sleep(3) + + # Check if we are already logged in + current_url = self.driver.current_url.lower() + if all(kw not in current_url for kw in ("login", "signin", "sign-in")): + logger.info("Already logged in (session restored from profile).") + return + + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + logger.info(f"Login attempt {attempt}/{max_attempts} …") + + # Find input fields + email_input = self._try_selectors([ + (By.CSS_SELECTOR, "input[name='email']"), + (By.CSS_SELECTOR, "input[type='email']"), + (By.XPATH, "//input[@placeholder[contains(translate(.,'EMAIL','email'),'email')]]"), + (By.XPATH, "//input[@autocomplete='username']"), + (By.XPATH, "//input[@autocomplete='email']"), + ], timeout=5) + + if email_input is None: + self._save_screenshot("login_page_not_found.png") + raise RuntimeError("Could not find email input on the login page.") + + self._type_humanlike(email_input, email) + logger.info("Email entered.") + + pwd_input = self._wait( + By.CSS_SELECTOR, "input[type='password']", timeout=10 + ) + self._type_humanlike(pwd_input, password) + logger.info("Password entered.") + + submit = self._try_selectors([ + (By.CSS_SELECTOR, "button[type='submit']"), + (By.XPATH, "//button[contains(translate(.,'LOGIN','login'),'login') or contains(translate(.,'SIGN IN','sign in'),'sign in')]"), + (By.CSS_SELECTOR, "button.ant-btn-primary"), + ], timeout=10) + + if submit is None: + raise RuntimeError("Could not find login submit button.") + + self._click(submit) + logger.info("Login form submitted.") + time.sleep(5) + + # Check if login failed (e.g. error message displayed) + try: + err = WebDriverWait(self.driver, 5).until( + EC.visibility_of_element_located(( + By.XPATH, + "//*[self::div or self::span or self::p or contains(@class,'error') or contains(@class,'message')][contains(text(), 'Login failed') or contains(text(), 'invalid') or contains(text(), 'incorrect')]" + )) + ) + logger.warning(f"Login failed message detected: {err.text}") + if attempt < max_attempts: + sleep_time = 45 * attempt + logger.info(f"Waiting {sleep_time} seconds before retrying login...") + time.sleep(sleep_time) + # Refresh the page to start clean + self.driver.get("https://app.factors.ai/") + time.sleep(3) + continue + else: + raise RuntimeError(f"Login rejected by Factors.AI after {max_attempts} attempts: {err.text}") + except TimeoutException: + # No failure message found, verify if we are logged in + current_url = self.driver.current_url.lower() + if all(kw not in current_url for kw in ("login", "signin", "sign-in")): + logger.info(f"Logged in successfully. URL: {self.driver.current_url}") + return + else: + # No error message, but still on login page. Wait and check again. + time.sleep(3) + if all(kw not in self.driver.current_url.lower() for kw in ("login", "signin", "sign-in")): + logger.info(f"Logged in successfully. URL: {self.driver.current_url}") + return + + raise RuntimeError("Still on login page after submission.") + + # ── Navigate to Account Profiles ────────────────────────────────────────── + + def _navigate_to_accounts(self) -> None: + """Find and open the Account Profiles page.""" + logger.info("Navigating to Account Profiles page …") + + # Derive base URL (e.g. https://app.factors.ai) from current URL + parts = self.driver.current_url.split("/") + base_url = "/".join(parts[:3]) # ['https:', '', 'app.factors.ai'] + + # 1. If we are already on the base URL, just wait for the download button to be present + from urllib.parse import urlparse + parsed = urlparse(self.driver.current_url) + path = parsed.path.rstrip("/") + if path in ("", "/", "/v2/accounts"): + logger.info("Already on the home/accounts URL. Waiting for the Accounts page to load...") + self._dismiss_upgrade_modal() + if self._download_button_present(timeout=25): + logger.info("Accounts page loaded successfully on the home page. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + + # 2. Check if the download button is already present (e.g. from a previous navigation) + self._dismiss_upgrade_modal() + if self._download_button_present(timeout=3): + logger.info("Already on the Account Profiles page. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + + # 3. Try clicking a sidebar / nav link first (fastest) + nav_candidates = [ + (By.ID, "fa-at-link--accounts"), + (By.XPATH, "//a[contains(translate(@href,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]"), + (By.XPATH, "//span[contains(translate(text(),'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]/ancestor::a"), + (By.XPATH, "//li[contains(translate(.,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz'),'account')]//a"), + ] + link = self._try_selectors(nav_candidates, timeout=5) + if link: + try: + self._click(link) + time.sleep(1) + if self._download_button_present(timeout=5): + logger.info(f"Accounts page found via nav. URL: {self.driver.current_url}. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + except Exception: + pass + + # 4. Try known URL patterns + for slug in [ + "/v2/accounts", + "/accounts", + "/", + ]: + url = base_url + slug + logger.info(f" Trying {url} …") + try: + self.driver.get(url) + if self._download_button_present(timeout=3): + logger.info(f"Accounts page found at: {url}. Waiting 5s for data loading/hydration...") + time.sleep(5) + return + except Exception: + pass + + # Last resort: dump screenshot and raise + self._save_screenshot("accounts_not_found.png") + raise RuntimeError( + "Could not navigate to the Account Profiles page. " + "Screenshot saved → tmp_download/accounts_not_found.png" + ) + + def _download_button_present(self, timeout: int = 6) -> bool: + """Return True if the toolbar download button is visible on the page.""" + combined_selector = ( + "button#fa-at-btn--export-csv, " + "button[id*='export-csv'], " + "[class*='profileToolbarIconBtn'], " + "button[title*='Download'], " + "button[title*='Export']" + ) + try: + WebDriverWait(self.driver, timeout).until( + EC.presence_of_element_located((By.CSS_SELECTOR, combined_selector)) + ) + return True + except TimeoutException: + # Fallback to xpath check + try: + WebDriverWait(self.driver, 1).until( + EC.presence_of_element_located(( + By.XPATH, + "//button[.//svg[@data-icon='download'] or contains(@class,'profileToolbarIconBtn')]" + )) + ) + return True + except TimeoutException: + return False + + def _click_download_button(self) -> None: + logger.info("Clicking the download toolbar button …") + + for attempt in range(3): + self._dismiss_upgrade_modal() + + # Find all profile toolbar buttons + candidates = self.driver.find_elements(By.CSS_SELECTOR, "[class*='profileToolbarIconBtn']") + btn = None + for c in candidates: + try: + html = c.get_attribute("outerHTML") or "" + if "download" in html or "fa-download" in html: + if c.is_displayed() and c.is_enabled(): + btn = c + break + except Exception: + pass + + if btn is None: + logger.warning(f"Download button not found on attempt {attempt + 1}") + time.sleep(2) + continue + + try: + self.driver.execute_script("arguments[0].scrollIntoView(true);", btn) + time.sleep(0.5) + self.driver.execute_script("arguments[0].click();", btn) + logger.info(f"Download button clicked via JS (attempt {attempt + 1}).") + except Exception as e: + logger.warning(f"Failed to click button on attempt {attempt + 1}: {e}") + time.sleep(2) + continue + + time.sleep(2.0) + + modals = self.driver.find_elements(By.CSS_SELECTOR, ".ant-modal-content") + if any(m.is_displayed() for m in modals): + logger.info("Modal/dropdown detected after click.") + return + logger.warning("Modal/dropdown did not open. Retrying click...") + + self._save_screenshot("download_btn_click_failed.png") + raise RuntimeError("Failed to open export modal after multiple clicks.") + + # ── Handle the export modal ─────────────────────────────────────────────── + + def _handle_modal(self, fields_to_select: list[str] | None) -> None: + logger.info("Waiting for CSV export modal …") + self._dismiss_upgrade_modal() + + # Find the visible modal that is actually the export modal + time.sleep(2) # Give time for the modal to open + modals = self.driver.find_elements(By.CSS_SELECTOR, ".ant-modal-content") + modal = None + for m in modals: + if m.is_displayed(): + text = m.text.lower() + if any(kw in text for kw in ("export", "field", "select", "column", "include")): + modal = m + break + + if modal is None: + # Fallback to waiting for the first visible modal + modal = WebDriverWait(self.driver, _ELEMENT_TIMEOUT).until( + EC.visibility_of_element_located((By.CSS_SELECTOR, ".ant-modal-content")) + ) + + time.sleep(1.0) # let the virtual list fully render + + # --- Optionally override field selection --- + if fields_to_select is not None: + # 1. Clear all + try: + clear_btn = modal.find_element( + By.XPATH, + ".//button[.//*[contains(text(),'Clear All')]]", + ) + self._click(clear_btn) + time.sleep(0.6) + logger.info("Cleared all field selections.") + except NoSuchElementException: + logger.warning("'Clear All' button not found — skipping.") + + # 2. Scroll through the virtual list and check each desired field + for field_val in fields_to_select: + self._select_field_in_modal(modal, field_val) + + # --- Click Export CSV --- + export_btn = None + + # 1. Look for any button with "export" or "csv" in text anywhere inside the modal + try: + buttons = modal.find_elements(By.TAG_NAME, "button") + for btn in buttons: + text = (btn.text or "").lower() + if "export" in text or "csv" in text: + export_btn = btn + break + except Exception: + pass + + # 2. Fallback to primary button inside the modal + if export_btn is None: + try: + export_btn = modal.find_element(By.CSS_SELECTOR, ".ant-btn-primary") + except NoSuchElementException: + pass + + if export_btn is None: + self._save_screenshot("export_btn_not_found.png") + raise RuntimeError("Could not find the 'Export CSV' button in modal.") + + self._click(export_btn) + logger.info("'Export CSV' button clicked.") + time.sleep(1.5) + self._dismiss_upgrade_modal() + + # def _select_field_in_modal(self, modal, field_val: str) -> None: + # """Scroll inside the modal virtual list until the checkbox is found, then check it.""" + # scrollable = modal.find_element(By.CSS_SELECTOR, ".rc-virtual-list-holder") + # max_scrolls = 20 + + # for _ in range(max_scrolls): + # try: + # # Find the label or span containing the exact field name + # label_el = modal.find_element( + # By.XPATH, + # f".//span[text()='{field_val}'] | .//label[contains(.,'{field_val}')] | .//*[text()='{field_val}']" + # ) + + # # Scroll it into view inside the virtual list container + # self.driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", label_el) + # time.sleep(0.1) + + # # Locate the parent/wrapper label and the checkbox input + # parent_label = label_el.find_element(By.XPATH, "./ancestor-or-self::label[contains(@class,'ant-checkbox-wrapper')]") + # checkbox = parent_label.find_element(By.CSS_SELECTOR, "input[type='checkbox']") + + # if not checkbox.is_selected(): + # self.driver.execute_script("arguments[0].click();", checkbox) + # time.sleep(0.2) + # logger.info(f"Selected field: {field_val}") + # return + # except (NoSuchElementException, StaleElementReferenceException): + # # Scroll down inside the modal list to reveal more items + # self.driver.execute_script( + # "arguments[0].scrollTop += 100;", scrollable + # ) + # time.sleep(1.5) + + # logger.warning(f"Field not found in modal: {field_val}") + + def _select_field_in_modal(self, modal, field_val: str) -> None: + """ + Select a field inside the Export CSV modal. + + Uses the modal's built-in search box instead of scrolling the + virtual list. This is much more reliable with Ant Design's + rc-virtual-list because only matching rows are rendered. + """ + + wait = WebDriverWait(self.driver, 15) + + try: + # Locate search input + search_box = wait.until( + lambda d: modal.find_element( + By.CSS_SELECTOR, + "input[placeholder='Search properties']" + ) + ) + + # Clear any previous search + search_box.click() + search_box.send_keys(Keys.CONTROL, "a") + search_box.send_keys(Keys.DELETE) + + # Type desired field + search_box.send_keys(field_val) + + # Wait until matching checkbox appears + checkbox = wait.until( + lambda d: modal.find_element( + By.XPATH, + f""" + .//label[ + .//*[normalize-space(text())="{field_val}"] + ]//input[@type='checkbox'] + """ + ) + ) + + # Tick if needed + if not checkbox.is_selected(): + self.driver.execute_script( + "arguments[0].click();", + checkbox + ) + + wait.until(lambda d: checkbox.is_selected()) + + logger.info(f"Selected field: {field_val}") + + except TimeoutException: + logger.warning(f"Field not found in modal: {field_val}") + + finally: + # Clear search so next iteration starts fresh + try: + search_box = modal.find_element( + By.CSS_SELECTOR, + "input[placeholder='Search properties']" + ) + + search_box.send_keys(Keys.CONTROL, "a") + search_box.send_keys(Keys.DELETE) + + wait.until( + lambda d: search_box.get_attribute("value") == "" + ) + + except Exception: + pass + + # ── Wait for the CSV download to finish ─────────────────────────────────── + + def _wait_for_download(self, timeout: int = _DOWNLOAD_TIMEOUT) -> str: + logger.info(f"Waiting up to {timeout}s for CSV download …") + deadline = time.time() + timeout + + while time.time() < deadline: + self._dismiss_upgrade_modal() + + all_csvs = glob.glob(os.path.join(self.download_dir, "*.csv")) + in_flight = glob.glob(os.path.join(self.download_dir, "*.crdownload")) + + logger.info(f"Checking downloads. CSVs: {len(all_csvs)}, in-flight: {len(in_flight)}") + + complete = [f for f in all_csvs if not f.endswith(".crdownload")] + if complete and not in_flight: + latest = max(complete, key=os.path.getmtime) + logger.info(f"Download complete → {latest}") + return latest + + time.sleep(1) + + self._save_screenshot("download_timeout.png") + raise TimeoutError( + f"CSV download did not complete within {timeout} seconds." + ) + + # ── Utilities ───────────────────────────────────────────────────────────── + + def _save_screenshot(self, filename: str) -> None: + if self.driver: + path = os.path.join(self.download_dir, filename) + try: + self.driver.save_screenshot(path) + logger.info(f"Screenshot saved → {path}") + except Exception: + pass + + def _cleanup_download_dir(self) -> None: + """Remove any leftover CSVs from previous runs.""" + for f in glob.glob(os.path.join(self.download_dir, "*.csv")): + try: + os.remove(f) + except OSError: + pass + + # ── Project switching and Modals ────────────────────────────────────────── + + def _dismiss_upgrade_modal(self) -> None: + """Find and dismiss the 'Upgrade to experience the full power of Factors' / 'Go beyond signals' modal if present.""" + try: + modals = self.driver.find_elements( + By.XPATH, + "//div[contains(@class, 'ant-modal-content')][.//*[contains(text(), 'Upgrade to experience') or contains(text(), 'Go beyond signals') or contains(text(), 'Book a Demo')]]" + ) + for m in modals: + if m.is_displayed(): + logger.info("Upgrade/Paywall modal detected! Dismissing...") + try: + close_btn = m.find_element( + By.CSS_SELECTOR, + "button.ant-modal-close, button[aria-label='Close'], .ant-modal-close-x, button[class*='close']" + ) + self._click(close_btn) + logger.info("Successfully dismissed upgrade modal.") + time.sleep(1) + except Exception as click_err: + logger.warning(f"Could not click close button on upgrade modal: {click_err}") + except Exception as e: + logger.debug(f"Error checking/dismissing upgrade modal: {e}") + + def _dismiss_popups(self) -> None: + """Attempt to dismiss any promotional or feedback modals that might block interaction.""" + self._dismiss_upgrade_modal() + logger.info("Checking for any blocking popups/modals to dismiss...") + + # 1. Look for G2 review popup / "Don't show this again" buttons + try: + g2_buttons = self.driver.find_elements( + By.XPATH, + "//button[contains(translate(text(), 'DONT SHOW', 'dont show'), 'dont show') or contains(., \"Don't show this again\")]" + ) + for btn in g2_buttons: + if btn.is_displayed(): + logger.info("Dismissing G2 / 'Don't show this again' popup.") + self._click(btn) + time.sleep(1) + except Exception as e: + logger.debug(f"Error checking/dismissing G2 popup: {e}") + + # 2. Look for general modal close buttons (e.g. ant-modal-close, ant-modal-close-x) + try: + close_buttons = self.driver.find_elements( + By.CSS_SELECTOR, + ".ant-modal-close, button.ant-modal-close, .ant-modal-close-x, [class*='ant-modal-close']" + ) + for btn in close_buttons: + if btn.is_displayed(): + # We do NOT want to close the project switch modal or fields modal! + parent_modal_text = "" + try: + parent_modal = btn.find_element(By.XPATH, "./ancestor::div[contains(@class, 'ant-modal-content')]") + parent_modal_text = parent_modal.text.lower() + except Exception: + pass + + if "switch" in parent_modal_text or "export" in parent_modal_text or "csv" in parent_modal_text: + logger.info("Skipping modal close for functional modal (switch project/export).") + continue + + logger.info("Clicking modal close button to dismiss popup.") + self._click(btn) + time.sleep(1) + except Exception as e: + logger.debug(f"Error dismissing modal popups: {e}") + + def _switch_project(self, target_project: str) -> None: + """Switch the current project to the target_project if not already active.""" + if not target_project: + logger.info("No target project specified; skipping switch.") + return + + # Dismiss any initially shown modals/popups + self._dismiss_popups() + + def clean_name(name: str) -> str: + return name.replace(" ", "").replace("-", "").replace("_", "").lower().strip() + + clean_target = clean_name(target_project) + + logger.info(f"Ensuring target project is set to: '{target_project}'") + + # 1. Locate the main project dropdown button + try: + dropdown_btn = self._wait( + By.ID, "fa-at-dropdown--projects", timeout=20, condition="clickable" + ) + except TimeoutException: + self._save_screenshot("project_dropdown_not_found.png") + raise RuntimeError("Could not find the project dropdown button (#fa-at-dropdown--projects).") + + current_project_text = dropdown_btn.text + logger.info(f"Main project button text: {current_project_text!r}") + + # If it's already active on the top nav, we're done + if clean_target in clean_name(current_project_text): + logger.info(f"Project '{target_project}' is already active.") + return + + # 2. Click the dropdown button to open the project selector + logger.info("Clicking project dropdown button...") + self._click(dropdown_btn) + time.sleep(2) + + # 3. Check the active project inside the popover/modal + try: + active_proj_div = self._wait( + By.CSS_SELECTOR, + "div[class*='ProjectModal__active_project_div']", + timeout=10, + condition="clickable" + ) + except TimeoutException: + active_proj_div = None + + if active_proj_div: + active_proj_text = active_proj_div.text + logger.info(f"Active project in modal/dropdown menu: {active_proj_text!r}") + + if clean_target in clean_name(active_proj_text): + logger.info(f"Project '{target_project}' is already selected in the active div.") + # Close the popover by pressing escape + self.driver.find_element(By.TAG_NAME, "body").send_keys(Keys.ESCAPE) + time.sleep(1) + return + + # Click active project div to reveal the project list + logger.info("Clicking active project div to open the list of all projects...") + self._click(active_proj_div) + time.sleep(2) + + # 4. Search/filter using the search input + try: + search_input = self.driver.find_element(By.CSS_SELECTOR, "input[placeholder='Search Project']") + search_input.clear() + search_input.send_keys(target_project) + logger.info(f"Typed '{target_project}' into project search input.") + time.sleep(1.5) + except NoSuchElementException: + logger.info("Project list search input not found; scanning list directly.") + + # Find the project item that matches the target project + project_items = self.driver.find_elements( + By.CSS_SELECTOR, "div[class*='ProjectModal__project_item']" + ) + target_item = None + for item in project_items: + item_text = item.text + if clean_target in clean_name(item_text): + target_item = item + break + + if not target_item: + self._save_screenshot("project_item_not_found.png") + raise RuntimeError( + f"Could not find project '{target_project}' in the project list. " + "Screenshot saved → tmp_download/project_item_not_found.png" + ) + + logger.info(f"Selecting project item: {target_item.text!r}") + self._click(target_item) + time.sleep(2) + + # 5. Handle the project switch confirmation modal if it appears + try: + switch_modal = WebDriverWait(self.driver, 5).until( + EC.visibility_of_element_located(( + By.XPATH, + "//div[contains(@class, 'ant-modal-content')][.//h4[contains(translate(text(), 'SWITCH', 'switch'), 'switch')]]" + )) + ) + logger.info("Switch confirmation modal appeared.") + switch_btn = switch_modal.find_element( + By.XPATH, + ".//button[contains(@class, 'ant-btn-primary') or .//span[text()='Switch']]" + ) + self._click(switch_btn) + logger.info("Clicked 'Switch' on confirmation modal.") + time.sleep(5) # Wait for page to reload/redirect + except TimeoutException: + logger.info("No switch confirmation modal appeared; switch may have been instant.") + + # Dismiss any popups after switching + self._dismiss_popups() + + # ── Public entry point ──────────────────────────────────────────────────── + + def run( + self, + email: str, + password: str, + target_project: str = "Inbound-Tech4Biz", + fields_to_select: list[str] | None = None, + ) -> str: + """ + Full end-to-end automation run. + + Returns: + Path to the freshly downloaded CSV file. + """ + self._cleanup_download_dir() + + try: + self._setup_driver() + self.login(email, password) + self._switch_project(target_project) + self._navigate_to_accounts() + self._click_download_button() + self._handle_modal(fields_to_select) + return self._wait_for_download() + + except Exception as exc: + self._save_screenshot("scraper_failure.png") + logger.error(f"Scraper failed: {exc}", exc_info=True) + raise + + finally: + if self.driver: + self.driver.quit() + logger.info("Browser closed.")