diff --git a/Dockerfile b/Dockerfile index 216891f..ac3d3a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,8 @@ # Use Python 3.11 slim image (revert from 3.12) FROM python:3.11-slim -# Install cron and other dependencies +# Install system dependencies (remove cron) RUN apt-get update && apt-get install -y \ - cron \ gcc \ curl \ && rm -rf /var/lib/apt/lists/* @@ -22,7 +21,8 @@ ENV DATABASE_PATH=/app/data/price_tracker.db \ DELAY_BETWEEN_REQUESTS=2 \ MAX_CONCURRENT_REQUESTS=1 \ REQUEST_TIMEOUT=30 \ - RETRY_ATTEMPTS=3 + RETRY_ATTEMPTS=3 \ + WEBHOOK_SECRET=your-secret-key-here # Copy requirements first for better caching COPY requirements.txt . @@ -41,178 +41,13 @@ RUN mkdir -p /app/data /var/log && \ # Copy application code COPY . . -# Create the daily scraper script -RUN echo '#!/usr/bin/env python3\n\ -import sys\n\ -import os\n\ -import asyncio\n\ -import logging\n\ -from datetime import datetime\n\ -\n\ -# Configure logging\n\ -logging.basicConfig(\n\ - level=logging.INFO,\n\ - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",\n\ - handlers=[\n\ - logging.FileHandler("/var/log/price_scraper.log"),\n\ - logging.StreamHandler()\n\ - ]\n\ -)\n\ -\n\ -logger = logging.getLogger(__name__)\n\ -\n\ -async def main():\n\ - try:\n\ - from src.config import Config\n\ - from src.database import DatabaseManager\n\ - from src.scraper_manager import ScraperManager\n\ - from src.notification import NotificationManager\n\ - from src.shopping_list import AutoShoppingListGenerator\n\ - \n\ - logger.info("Starting scheduled price scraping")\n\ - \n\ - config = Config()\n\ - if config.has_config_error():\n\ - logger.error(f"Configuration error: {config.get_config_error()}")\n\ - return\n\ - \n\ - db_manager = DatabaseManager(config.database_path)\n\ - scraper_manager = ScraperManager(config)\n\ - notification_manager = NotificationManager(config)\n\ - shopping_list_generator = AutoShoppingListGenerator(db_manager)\n\ - \n\ - products = db_manager.get_all_products()\n\ - if not products:\n\ - logger.warning("No products found to scrape")\n\ - return\n\ - \n\ - logger.info(f"Scraping {len(products)} products")\n\ - results = await scraper_manager.scrape_all_products(products)\n\ - \n\ - total = sum(len(sites) for sites in results.values())\n\ - successful = sum(1 for sites in results.values() for result in sites.values() if result["success"])\n\ - failed = total - successful\n\ - \n\ - logger.info(f"Scraping complete: {successful}/{total} successful")\n\ - \n\ - # Save results and collect price alerts\n\ - price_alerts = []\n\ - for product_id, site_results in results.items():\n\ - product = db_manager.get_product(product_id)\n\ - for site_name, result in site_results.items():\n\ - if result["success"]:\n\ - # Save to database\n\ - db_manager.save_price_history(\n\ - product_id=product_id,\n\ - site_name=site_name,\n\ - price=result["price"],\n\ - availability=result.get("availability", True),\n\ - timestamp=datetime.now()\n\ - )\n\ - \n\ - # Check for price alerts\n\ - if product and product.get("target_price") and result["price"] <= product["target_price"]:\n\ - price_alerts.append({\n\ - "product": product,\n\ - "site": site_name,\n\ - "current_price": result["price"],\n\ - "target_price": product["target_price"],\n\ - "url": result.get("url", "")\n\ - })\n\ - \n\ - # Send price alerts if any\n\ - if price_alerts:\n\ - alert_message = "Price Alerts:\\n\\n"\n\ - for alert in price_alerts:\n\ - alert_message += f"🎯 {alert[\"product\"][\"name\"]}\\n"\n\ - alert_message += f" Store: {alert[\"site\"]}\\n"\n\ - alert_message += f" Price: £{alert[\"current_price\"]} (Target: £{alert[\"target_price\"]})\\n"\n\ - alert_message += f" URL: {alert[\"url\"]}\\n\\n"\n\ - \n\ - await notification_manager.send_notification(\n\ - subject=f"Price Alert: {len(price_alerts)} item(s) on sale!",\n\ - message=alert_message\n\ - )\n\ - logger.info(f"Sent price alerts for {len(price_alerts)} items")\n\ - \n\ - # Generate and send daily shopping list\n\ - try:\n\ - shopping_lists = shopping_list_generator.generate_all_shopping_lists()\n\ - if shopping_lists:\n\ - shopping_message = "Daily Shopping List (Best Prices):\\n\\n"\n\ - total_savings = 0\n\ - \n\ - for store_name, store_list in shopping_lists.items():\n\ - if store_list.items:\n\ - shopping_message += f"🏪 {store_name.upper()}:\\n"\n\ - store_total = 0\n\ - for item in store_list.items:\n\ - shopping_message += f" • {item.product_name} - £{item.current_price}\\n"\n\ - store_total += item.current_price\n\ - if item.savings_amount > 0:\n\ - total_savings += item.savings_amount\n\ - shopping_message += f" Subtotal: £{store_total:.2f}\\n\\n"\n\ - \n\ - if total_savings > 0:\n\ - shopping_message += f"💰 Total Savings: £{total_savings:.2f}\\n"\n\ - \n\ - await notification_manager.send_notification(\n\ - subject="Daily Shopping List - Best Prices",\n\ - message=shopping_message\n\ - )\n\ - logger.info("Sent daily shopping list")\n\ - except Exception as e:\n\ - logger.error(f"Failed to generate shopping list: {e}")\n\ - \n\ - # Send scraping summary\n\ - summary_message = f"Daily Price Scraping Summary:\\n\\n"\n\ - summary_message += f"📊 Products scraped: {len(products)}\\n"\n\ - summary_message += f"✅ Successful: {successful}\\n"\n\ - summary_message += f"❌ Failed: {failed}\\n"\n\ - summary_message += f"🎯 Price alerts: {len(price_alerts)}\\n"\n\ - summary_message += f"🕐 Completed at: {datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}\\n"\n\ - \n\ - await notification_manager.send_notification(\n\ - subject="Daily Price Scraping Complete",\n\ - message=summary_message\n\ - )\n\ - logger.info("Sent scraping summary")\n\ - \n\ - except Exception as e:\n\ - logger.error(f"Scheduled scraping failed: {str(e)}", exc_info=True)\n\ - \n\ - # Send error notification\n\ - try:\n\ - from src.config import Config\n\ - from src.notification import NotificationManager\n\ - config = Config()\n\ - notification_manager = NotificationManager(config)\n\ - await notification_manager.send_notification(\n\ - subject="Price Scraping Failed",\n\ - message=f"Daily price scraping failed with error:\\n\\n{str(e)}"\n\ - )\n\ - except:\n\ - pass # If notification also fails, just log\n\ -\n\ -if __name__ == "__main__":\n\ - asyncio.run(main())\n\ -' > /app/daily_scraper.py && chmod +x /app/daily_scraper.py - -# Create cron job - runs daily at 8 AM -RUN echo "0 15 * * * cd /app && python daily_scraper.py >> /var/log/cron.log 2>&1" > /etc/cron.d/price-tracker -RUN chmod 0644 /etc/cron.d/price-tracker -RUN crontab /etc/cron.d/price-tracker - -# Create startup script that ensures directories exist and have correct permissions +# Create startup script without cron RUN echo '#!/bin/bash\n\ # Ensure data directory exists and has correct permissions\n\ mkdir -p /app/data /var/log\n\ chown -R tracker:tracker /app/data /var/log\n\ chmod 755 /app/data /var/log\n\ \n\ -# Start cron in background\n\ -cron\n\ -\n\ # Switch to non-root user and start web server\n\ exec su tracker -c "python main.py --mode web"\n\ ' > /app/start.sh && chmod +x /app/start.sh diff --git a/src/web_ui.py b/src/web_ui.py index 76df37e..ab697bd 100644 --- a/src/web_ui.py +++ b/src/web_ui.py @@ -13,6 +13,10 @@ import plotly import plotly.graph_objs as go import pandas as pd import os +import hmac +import hashlib +from functools import wraps +from flask import request, jsonify from .database import DatabaseManager from .config import Config @@ -474,4 +478,240 @@ def create_app(): except Exception as e: return jsonify({'error': str(e)}), 500 + # Add webhook authentication decorator + def webhook_auth_required(f): + @wraps(f) + def decorated_function(*args, **kwargs): + # Get webhook secret from environment + webhook_secret = os.environ.get('WEBHOOK_SECRET', 'your-secret-key-here') + + # Check for secret in header or query parameter + provided_secret = request.headers.get('X-Webhook-Secret') or request.args.get('secret') + + if not provided_secret or provided_secret != webhook_secret: + return jsonify({'error': 'Unauthorized'}), 401 + + return f(*args, **kwargs) + return decorated_function + + # Add webhook routes + @app.route('/webhook/scrape', methods=['POST', 'GET']) + @webhook_auth_required + def webhook_scrape(): + """Webhook endpoint to trigger price scraping""" + try: + # Run scraping in background + import asyncio + from .scraper_manager import ScraperManager + from .notification import NotificationManager + + async def run_scrape(): + try: + logger.info("Webhook triggered price scraping") + + config = Config() + if config.has_config_error(): + logger.error(f"Configuration error: {config.get_config_error()}") + return {'error': 'Configuration error'} + + db_manager = DatabaseManager(config.database_path) + scraper_manager = ScraperManager(config) + notification_manager = NotificationManager(config) + + products = db_manager.get_all_products() + if not products: + logger.warning("No products found to scrape") + return {'message': 'No products to scrape'} + + logger.info(f"Scraping {len(products)} products") + results = await scraper_manager.scrape_all_products(products) + + total = sum(len(sites) for sites in results.values()) + successful = sum(1 for sites in results.values() for result in sites.values() if result['success']) + failed = total - successful + + logger.info(f"Scraping complete: {successful}/{total} successful") + + # Save results and collect price alerts + price_alerts = [] + for product_id, site_results in results.items(): + product = db_manager.get_product(product_id) + for site_name, result in site_results.items(): + if result['success']: + # Save to database + db_manager.save_price_history( + product_id=product_id, + site_name=site_name, + price=result['price'], + availability=result.get('availability', True), + timestamp=datetime.now() + ) + + # Check for price alerts + if product and product.get('target_price') and result['price'] <= product['target_price']: + price_alerts.append({ + 'product': product, + 'site': site_name, + 'current_price': result['price'], + 'target_price': product['target_price'], + 'url': result.get('url', '') + }) + + # Send price alerts if any + if price_alerts: + alert_message = "Price Alerts:\n\n" + for alert in price_alerts: + alert_message += f"🎯 {alert['product']['name']}\n" + alert_message += f" Store: {alert['site']}\n" + alert_message += f" Price: £{alert['current_price']} (Target: £{alert['target_price']})\n" + alert_message += f" URL: {alert['url']}\n\n" + + await notification_manager.send_notification( + subject=f"Price Alert: {len(price_alerts)} item(s) on sale!", + message=alert_message + ) + logger.info(f"Sent price alerts for {len(price_alerts)} items") + + # Send scraping summary + summary_message = f"Daily Price Scraping Summary:\n\n" + summary_message += f"📊 Products scraped: {len(products)}\n" + summary_message += f"✅ Successful: {successful}\n" + summary_message += f"❌ Failed: {failed}\n" + summary_message += f"🎯 Price alerts: {len(price_alerts)}\n" + summary_message += f"🕐 Completed at: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n" + + await notification_manager.send_notification( + subject="Daily Price Scraping Complete", + message=summary_message + ) + logger.info("Sent scraping summary") + + return { + 'message': 'Scraping completed successfully', + 'total_products': len(products), + 'successful': successful, + 'failed': failed, + 'price_alerts': len(price_alerts) + } + + except Exception as e: + logger.error(f"Webhook scraping failed: {str(e)}", exc_info=True) + + # Send error notification + try: + await notification_manager.send_notification( + subject="Price Scraping Failed", + message=f"Daily price scraping failed with error:\n\n{str(e)}" + ) + except: + pass + + return {'error': str(e)} + + # Run the async function + result = asyncio.run(run_scrape()) + + return jsonify(result) + + except Exception as e: + logger.error(f"Webhook error: {str(e)}", exc_info=True) + return jsonify({'error': str(e)}), 500 + + @app.route('/webhook/shopping-list', methods=['POST', 'GET']) + @webhook_auth_required + def webhook_shopping_list(): + """Webhook endpoint to send daily shopping list""" + try: + from .shopping_list import AutoShoppingListGenerator + from .notification import NotificationManager + + config = Config() + if config.has_config_error(): + return jsonify({'error': 'Configuration error'}), 500 + + db_manager = DatabaseManager(config.database_path) + shopping_list_generator = AutoShoppingListGenerator(db_manager) + notification_manager = NotificationManager(config) + + # Generate shopping lists + shopping_lists = shopping_list_generator.generate_all_shopping_lists() + + if shopping_lists: + shopping_message = "Daily Shopping List (Best Prices):\n\n" + total_savings = 0 + + for store_name, store_list in shopping_lists.items(): + if store_list.items: + shopping_message += f"🏪 {store_name.upper()}:\n" + store_total = 0 + for item in store_list.items: + shopping_message += f" • {item.product_name} - £{item.current_price}\n" + store_total += item.current_price + if item.savings_amount > 0: + total_savings += item.savings_amount + shopping_message += f" Subtotal: £{store_total:.2f}\n\n" + + if total_savings > 0: + shopping_message += f"💰 Total Savings: £{total_savings:.2f}\n" + + # Send email using asyncio + async def send_email(): + await notification_manager.send_notification( + subject="Daily Shopping List - Best Prices", + message=shopping_message + ) + + asyncio.run(send_email()) + + return jsonify({ + 'message': 'Shopping list sent successfully', + 'stores': list(shopping_lists.keys()), + 'total_savings': total_savings + }) + else: + return jsonify({'message': 'No shopping lists generated'}) + + except Exception as e: + logger.error(f"Shopping list webhook error: {str(e)}", exc_info=True) + return jsonify({'error': str(e)}), 500 + + @app.route('/webhook/scrape-and-list', methods=['POST', 'GET']) + @webhook_auth_required + def webhook_scrape_and_list(): + """Webhook endpoint to scrape prices AND send shopping list""" + try: + # First trigger scraping + scrape_response = webhook_scrape() + scrape_data = scrape_response.get_json() + + if 'error' in scrape_data: + return jsonify({'scraping_error': scrape_data['error']}), 500 + + # Then send shopping list + list_response = webhook_shopping_list() + list_data = list_response.get_json() + + return jsonify({ + 'message': 'Scraping and shopping list completed', + 'scraping': scrape_data, + 'shopping_list': list_data + }) + + except Exception as e: + logger.error(f"Combined webhook error: {str(e)}", exc_info=True) + return jsonify({'error': str(e)}), 500 + + @app.route('/webhook/health', methods=['GET']) + def webhook_health(): + """Health check endpoint for webhooks""" + return jsonify({ + 'status': 'healthy', + 'timestamp': datetime.now().isoformat(), + 'endpoints': [ + '/webhook/scrape', + '/webhook/shopping-list', + '/webhook/scrape-and-list' + ] + }) + return app