From 448af85a389421e3a96f1eebf4d219209cc5ae73 Mon Sep 17 00:00:00 2001
From: Oli Passey <oli@hedgerows.net>
Date: Fri, 27 Jun 2025 11:12:24 +0100
Subject: [PATCH] pipeline

---
 AZURE-DEVOPS-SETUP.md        | 214 +++++++++++++++++++++++++++++
 azure-pipelines-advanced.yml | 251 +++++++++++++++++++++++++++++++++++
 azure-pipelines.yml          |  79 +++++++++++
 src/uk_scraper_new.py        |   0
 4 files changed, 544 insertions(+)
 create mode 100644 AZURE-DEVOPS-SETUP.md
 create mode 100644 azure-pipelines-advanced.yml
 create mode 100644 azure-pipelines.yml
 create mode 100644 src/uk_scraper_new.py

diff --git a/AZURE-DEVOPS-SETUP.md b/AZURE-DEVOPS-SETUP.md
new file mode 100644
index 0000000..ba1c916
--- /dev/null
+++ b/AZURE-DEVOPS-SETUP.md
@@ -0,0 +1,214 @@
+# Azure DevOps Pipeline Setup Guide
+
+This guide will help you set up the Azure DevOps pipeline to automatically build and push your Price Tracker Docker images to your registry at `dock.ptslondon.co.uk`.
+
+## Prerequisites
+
+1. Azure DevOps project with your Price Tracker repository
+2. Access to your Docker registry at `dock.ptslondon.co.uk`
+3. Admin permissions in Azure DevOps to create service connections
+
+## Step 1: Create Docker Registry Service Connection
+
+1. In your Azure DevOps project, go to **Project Settings** (bottom left)
+2. Under **Pipelines**, click **Service connections**
+3. Click **New service connection**
+4. Select **Docker Registry** and click **Next**
+5. Choose **Others** for registry type
+6. Fill in the details:
+   - **Docker Registry**: `https://dock.ptslondon.co.uk`
+   - **Docker ID**: Your registry username
+   - **Docker Password**: Your registry password
+   - **Service connection name**: `dock-ptslondon-connection`
+7. Check **Grant access permission to all pipelines**
+8. Click **Save**
+
+## Step 2: Choose Your Pipeline
+
+You have two pipeline options:
+
+### Basic Pipeline (`azure-pipelines.yml`)
+- Simple build and push
+- Basic testing
+- Good for getting started
+
+### Advanced Pipeline (`azure-pipelines-advanced.yml`)
+- Multi-stage deployment
+- Security scanning with Trivy
+- Separate dev/prod environments
+- More comprehensive testing
+
+## Step 3: Create the Pipeline
+
+1. In Azure DevOps, go to **Pipelines** → **Pipelines**
+2. Click **New pipeline**
+3. Select **Azure Repos Git** (or your source)
+4. Select your repository
+5. Choose **Existing Azure Pipelines YAML file**
+6. Select the pipeline file:
+   - `/azure-pipelines.yml` (basic)
+   - `/azure-pipelines-advanced.yml` (advanced)
+7. Click **Continue**
+8. Review the pipeline and click **Run**
+
+## Step 4: Configure Environments (Advanced Pipeline Only)
+
+If using the advanced pipeline, create environments:
+
+1. Go to **Pipelines** → **Environments**
+2. Click **New environment**
+3. Create two environments:
+   - **Name**: `price-tracker-dev`
+   - **Name**: `price-tracker-prod`
+4. For each environment, you can add approval gates:
+   - Go to the environment
+   - Click the three dots → **Approvals and checks**
+   - Add **Approvals** for production
+
+## Step 5: Pipeline Configuration
+
+The pipeline is configured to:
+
+### Triggers
+- **Push to main**: Builds and deploys to production
+- **Push to develop**: Builds and deploys to development
+- **Pull requests**: Builds and tests only
+
+### Build Process
+1. Checkout source code
+2. Build Docker image
+3. Run container tests
+4. Push to registry (if not PR)
+5. Security scan (optional)
+6. Deploy to appropriate environment
+
+### Variables Used
+- `dockerRegistryServiceConnection`: Service connection name
+- `imageRepository`: `price-tracker`
+- `containerRegistry`: `dock.ptslondon.co.uk`
+- `tag`: Uses build ID for versioning
+
+## Step 6: Customize for Your Needs
+
+### Registry Settings
+If your registry requires different settings, update these variables in the pipeline:
+
+```yaml
+variables:
+  dockerRegistryServiceConnection: 'your-connection-name'
+  imageRepository: 'your-repo-name'
+  containerRegistry: 'dock.ptslondon.co.uk'
+```
+
+### Branch Strategy
+Current setup:
+- `main` branch → Production deployment
+- `develop` branch → Development deployment
+
+To change this, modify the trigger and condition sections.
+
+### Security Scanning
+The advanced pipeline includes Trivy security scanning. To disable:
+- Remove the `SecurityScan` stage
+- Remove it from the `dependsOn` lists
+
+## Step 7: Verify the Setup
+
+1. Make a small change to your code
+2. Push to the `develop` branch (for testing)
+3. Watch the pipeline run in Azure DevOps
+4. Verify the image appears in your registry
+5. Check that the application deploys correctly
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Service Connection Failed**
+   - Verify registry credentials
+   - Check registry URL format
+   - Ensure registry is accessible from Azure
+
+2. **Docker Build Failed**
+   - Check Dockerfile syntax
+   - Verify all required files are in repository
+   - Check build logs for specific errors
+
+3. **Push to Registry Failed**
+   - Verify service connection permissions
+   - Check registry quota/space
+   - Ensure repository name is correct
+
+4. **Tests Failed**
+   - Check application startup logs
+   - Verify port mappings
+   - Ensure dependencies are available
+
+### Debug Tips
+
+1. **Enable verbose logging**:
+   ```yaml
+   - script: |
+       set -x  # Enable debug mode
+       # your commands here
+   ```
+
+2. **Add diagnostic steps**:
+   ```yaml
+   - script: |
+       docker images
+       docker ps -a
+       curl -v http://localhost:5000/
+   ```
+
+3. **Check container logs**:
+   ```yaml
+   - script: |
+       docker logs price-tracker-test
+   ```
+
+## Pipeline Outputs
+
+### Successful Run Produces:
+- Docker image tagged with build ID
+- Docker image tagged as `latest`
+- Images pushed to `dock.ptslondon.co.uk/price-tracker:BUILD_ID`
+- Deployment artifacts
+- Test results
+
+### Image Tags
+- `dock.ptslondon.co.uk/price-tracker:latest` - Latest build
+- `dock.ptslondon.co.uk/price-tracker:12345` - Specific build ID
+
+## Production Deployment
+
+For production deployment, the pipeline:
+1. Only runs on `main` branch
+2. Requires environment approval (if configured)
+3. Runs security scans
+4. Performs health checks
+5. Can include notifications
+
+## Monitoring
+
+Monitor your pipeline:
+1. **Pipeline runs**: Azure DevOps → Pipelines → Runs
+2. **Environment status**: Azure DevOps → Pipelines → Environments
+3. **Registry images**: Check your registry dashboard
+4. **Application logs**: From deployed containers
+
+## Next Steps
+
+1. Set up monitoring and alerting for your deployed application
+2. Configure backup strategies for your registry
+3. Set up staging environments for testing
+4. Add integration tests to the pipeline
+5. Configure notification webhooks for deployment status
+
+## Security Best Practices
+
+1. **Never commit secrets** - Use Azure DevOps secure variables
+2. **Use service connections** - Don't embed credentials
+3. **Scan images** - Enable vulnerability scanning
+4. **Limit permissions** - Use least privilege access
+5. **Monitor access** - Regular audit of service connections
diff --git a/azure-pipelines-advanced.yml b/azure-pipelines-advanced.yml
new file mode 100644
index 0000000..af45048
--- /dev/null
+++ b/azure-pipelines-advanced.yml
@@ -0,0 +1,251 @@
+trigger:
+  branches:
+    include:
+    - main
+    - develop
+  paths:
+    include:
+    - src/*
+    - templates/*
+    - requirements.txt
+    - Dockerfile
+    - config.json
+    - main.py
+
+pr:
+  branches:
+    include:
+    - main
+  paths:
+    include:
+    - src/*
+    - templates/*
+    - requirements.txt
+    - Dockerfile
+    - config.json
+    - main.py
+
+variables:
+  # Container registry service connection
+  dockerRegistryServiceConnection: 'dock-ptslondon-connection'
+  imageRepository: 'price-tracker'
+  containerRegistry: 'dock.ptslondon.co.uk'
+  dockerfilePath: '$(Build.SourcesDirectory)/Dockerfile'
+  
+  # Agent VM image name
+  vmImageName: 'ubuntu-latest'
+
+stages:
+- stage: Build
+  displayName: 'Build and Test'
+  jobs:
+  - job: Build
+    displayName: 'Build Docker Image'
+    pool:
+      vmImage: $(vmImageName)
+    variables:
+      tag: '$(Build.BuildId)'
+    steps:
+    - checkout: self
+      displayName: 'Checkout source code'
+
+    - task: Docker@2
+      displayName: 'Build Docker image'
+      inputs:
+        command: 'build'
+        repository: $(imageRepository)
+        dockerfile: $(dockerfilePath)
+        tags: |
+          $(tag)
+          latest
+
+    - script: |
+        echo "Running container tests..."
+        # Start container for testing
+        docker run -d --name price-tracker-test -p 5000:5000 $(imageRepository):$(tag)
+        
+        # Wait for container to be ready
+        echo "Waiting for container to start..."
+        for i in {1..30}; do
+          if curl -f http://localhost:5000/ > /dev/null 2>&1; then
+            echo "Container is ready!"
+            break
+          fi
+          echo "Waiting... ($i/30)"
+          sleep 2
+        done
+        
+        # Run basic health checks
+        echo "Running health checks..."
+        curl -f http://localhost:5000/ || (echo "Health check failed" && exit 1)
+        
+        # Cleanup
+        docker stop price-tracker-test
+        docker rm price-tracker-test
+        
+        echo "All tests passed!"
+      displayName: 'Test Docker container'
+
+    - task: Docker@2
+      displayName: 'Push to registry'
+      inputs:
+        command: 'push'
+        repository: $(imageRepository)
+        containerRegistry: $(dockerRegistryServiceConnection)
+        tags: |
+          $(tag)
+          latest
+      condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+
+- stage: SecurityScan
+  displayName: 'Security Scanning'
+  dependsOn: Build
+  condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'))
+  jobs:
+  - job: VulnerabilityScan
+    displayName: 'Vulnerability Scan'
+    pool:
+      vmImage: $(vmImageName)
+    variables:
+      tag: '$(Build.BuildId)'
+    steps:
+    - task: Docker@2
+      displayName: 'Pull image for scanning'
+      inputs:
+        command: 'pull'
+        arguments: '$(containerRegistry)/$(imageRepository):$(tag)'
+        containerRegistry: $(dockerRegistryServiceConnection)
+
+    - script: |
+        # Install Trivy
+        sudo apt-get update
+        sudo apt-get install wget apt-transport-https gnupg lsb-release -y
+        wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+        echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+        sudo apt-get update
+        sudo apt-get install trivy -y
+        
+        # Run vulnerability scan
+        trivy image --exit-code 0 --severity LOW,MEDIUM --format table $(containerRegistry)/$(imageRepository):$(tag)
+        trivy image --exit-code 1 --severity HIGH,CRITICAL --format table $(containerRegistry)/$(imageRepository):$(tag)
+      displayName: 'Run Trivy vulnerability scan'
+      continueOnError: true
+
+- stage: DeployDev
+  displayName: 'Deploy to Development'
+  dependsOn: 
+  - Build
+  - SecurityScan
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/develop'))
+  jobs:
+  - deployment: DeployToDev
+    displayName: 'Deploy to Development Environment'
+    environment: 'price-tracker-dev'
+    pool:
+      vmImage: $(vmImageName)
+    variables:
+      tag: '$(Build.BuildId)'
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - checkout: self
+          
+          - task: Docker@2
+            displayName: 'Pull latest image'
+            inputs:
+              command: 'pull'
+              arguments: '$(containerRegistry)/$(imageRepository):$(tag)'
+              containerRegistry: $(dockerRegistryServiceConnection)
+
+          - script: |
+              # Create deployment directory
+              mkdir -p ~/price-tracker-deployment
+              cd ~/price-tracker-deployment
+              
+              # Copy deployment files
+              cp $(Pipeline.Workspace)/s/docker-compose.yml .
+              cp $(Pipeline.Workspace)/s/config.json .
+              
+              # Update image tag in docker-compose
+              sed -i "s/price-tracker:latest/$(containerRegistry)\/$(imageRepository):$(tag)/g" docker-compose.yml
+              
+              # Deploy using docker-compose
+              docker-compose down || true
+              docker-compose up -d
+              
+              # Wait for deployment
+              sleep 10
+              
+              # Verify deployment
+              curl -f http://localhost:5000/ || (echo "Deployment verification failed" && exit 1)
+              
+              echo "Deployment to development completed successfully!"
+            displayName: 'Deploy to development'
+
+- stage: DeployProd
+  displayName: 'Deploy to Production'
+  dependsOn: 
+  - Build
+  - SecurityScan
+  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
+  jobs:
+  - deployment: DeployToProd
+    displayName: 'Deploy to Production Environment'
+    environment: 'price-tracker-prod'
+    pool:
+      vmImage: $(vmImageName)
+    variables:
+      tag: '$(Build.BuildId)'
+    strategy:
+      runOnce:
+        deploy:
+          steps:
+          - checkout: self
+          
+          - task: Docker@2
+            displayName: 'Pull latest image'
+            inputs:
+              command: 'pull'
+              arguments: '$(containerRegistry)/$(imageRepository):$(tag)'
+              containerRegistry: $(dockerRegistryServiceConnection)
+
+          - script: |
+              # Production deployment script
+              echo "Deploying to production..."
+              
+              # Create deployment directory
+              mkdir -p ~/price-tracker-production
+              cd ~/price-tracker-production
+              
+              # Copy deployment files
+              cp $(Pipeline.Workspace)/s/docker-compose.yml .
+              cp $(Pipeline.Workspace)/s/config.json .
+              
+              # Update image tag in docker-compose for production
+              sed -i "s/price-tracker:latest/$(containerRegistry)\/$(imageRepository):$(tag)/g" docker-compose.yml
+              
+              # Production-specific environment settings
+              export FLASK_ENV=production
+              
+              # Deploy with zero-downtime strategy
+              docker-compose pull
+              docker-compose up -d
+              
+              # Health check
+              sleep 15
+              for i in {1..10}; do
+                if curl -f http://localhost:5000/; then
+                  echo "Production deployment successful!"
+                  break
+                fi
+                echo "Waiting for production deployment... ($i/10)"
+                sleep 5
+              done
+            displayName: 'Deploy to production'
+
+          - script: |
+              # Post-deployment notifications (optional)
+              echo "Production deployment completed at $(date)"
+              # You can add notification webhooks here
+            displayName: 'Post-deployment tasks'
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
new file mode 100644
index 0000000..1ac280e
--- /dev/null
+++ b/azure-pipelines.yml
@@ -0,0 +1,79 @@
+trigger:
+  branches:
+    include:
+    - main
+    - develop
+  paths:
+    include:
+    - src/*
+    - templates/*
+    - requirements.txt
+    - Dockerfile
+    - config.json
+    - main.py
+
+variables:
+  # Container registry service connection established during pipeline creation
+  dockerRegistryServiceConnection: 'dock-ptslondon-connection'
+  imageRepository: 'price-tracker'
+  containerRegistry: 'dock.ptslondon.co.uk'
+  dockerfilePath: '$(Build.SourcesDirectory)/Dockerfile'
+  tag: '$(Build.BuildId)'
+  
+  # Agent VM image name
+  vmImageName: 'ubuntu-latest'
+
+stages:
+- stage: Build
+  displayName: Build and push stage
+  jobs:
+  - job: Build
+    displayName: Build
+    pool:
+      vmImage: $(vmImageName)
+    steps:
+    - task: Docker@2
+      displayName: Build and push Docker image
+      inputs:
+        command: buildAndPush
+        repository: $(imageRepository)
+        dockerfile: $(dockerfilePath)
+        containerRegistry: $(dockerRegistryServiceConnection)
+        tags: |
+          $(tag)
+          latest
+
+    # Optional: Run security scan on the image
+    - task: Docker@2
+      displayName: Run Trivy vulnerability scanner
+      inputs:
+        command: 'run'
+        arguments: '--rm -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/tmp/trivy aquasec/trivy image --exit-code 0 --severity HIGH,CRITICAL $(containerRegistry)/$(imageRepository):$(tag)'
+      continueOnError: true
+
+    # Optional: Test the built image
+    - task: Docker@2
+      displayName: Test Docker image
+      inputs:
+        command: 'run'
+        arguments: '--rm -d --name price-tracker-test -p 5001:5000 $(containerRegistry)/$(imageRepository):$(tag)'
+      continueOnError: true
+
+    - script: |
+        # Wait for container to start
+        sleep 10
+        # Test health endpoint
+        curl -f http://localhost:5001/ || echo "Health check failed"
+        # Cleanup test container
+        docker stop price-tracker-test || true
+      displayName: 'Health check test'
+      continueOnError: true
+
+    # Publish build artifacts
+    - task: PublishBuildArtifacts@1
+      displayName: 'Publish docker-compose and deployment files'
+      inputs:
+        PathtoPublish: '$(Build.SourcesDirectory)'
+        ArtifactName: 'deployment-files'
+        publishLocation: 'Container'
+      condition: succeeded()
diff --git a/src/uk_scraper_new.py b/src/uk_scraper_new.py
new file mode 100644
index 0000000..e69de29