feat: add web dashboard and metadata enrichment daemon

2026-01-23 17:49:48 +01:00
parent 47a8ceab32
commit 608af97a93
7 changed files with 497 additions and 2 deletions
--- a/Dockerfile.dashboard
+++ b/Dockerfile.dashboard
@@ -0,0 +1,10 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "dashboard/server.py"]
--- a/Dockerfile.metadata
+++ b/Dockerfile.metadata
@@ -0,0 +1,10 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD ["python", "src/metadata/fetcher.py"]
--- a/dashboard/public/index.html
+++ b/dashboard/public/index.html
@@ -0,0 +1,248 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Trading Intelligence Dashboard</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <link href="https://fonts.googleapis.com/css2?family=Outfit:wght@300;400;600;700&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: 'Outfit', sans-serif;
+            background-color: #0b0e14;
+            color: #e2e8f0;
+        }
+        .glass {
+            background: rgba(30, 41, 59, 0.5);
+            backdrop-filter: blur(12px);
+            border: 1px solid rgba(255, 255, 255, 0.1);
+            border-radius: 16px;
+        }
+        .glow {
+            box-shadow: 0 0 20px rgba(56, 189, 248, 0.2);
+        }
+        .sidebar-item:hover {
+            background: rgba(56, 189, 248, 0.1);
+            color: #38bdf8;
+        }
+        canvas {
+            max-width: 100%;
+        }
+    </style>
+</head>
+<body class="flex min-h-screen">
+
+    <!-- Sidebar -->
+    <aside class="w-64 glass m-4 flex flex-col hidden lg:flex">
+        <div class="p-6">
+            <h1 class="text-2xl font-bold bg-gradient-to-r from-sky-400 to-blue-500 bg-clip-text text-transparent">Antigravity Trade</h1>
+        </div>
+        <nav class="flex-1 px-4 space-y-2 mt-4">
+            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition">
+                <span class="mr-3">📊</span> Dashboard
+            </a>
+            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
+                <span class="mr-3">🌍</span> Global Markets
+            </a>
+            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
+                <span class="mr-3">🏢</span> Companies
+            </a>
+            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
+                <span class="mr-3">⚙️</span> Settings
+            </a>
+        </nav>
+        <div class="p-6 mt-auto">
+            <div class="glass p-4 text-xs text-sky-400 border-sky-500/30">
+                System Status: <span class="text-green-400">Live</span>
+            </div>
+        </div>
+    </aside>
+
+    <!-- Main Content -->
+    <main class="flex-1 p-8 overflow-y-auto">
+        <header class="flex justify-between items-center mb-8">
+            <div>
+                <h2 class="text-3xl font-bold">Market Overview</h2>
+                <p class="text-slate-400">Real-time trading analytics across multiple exchanges.</p>
+            </div>
+            <div class="flex space-x-4">
+                <select id="timeRange" class="glass px-4 py-2 text-sm outline-none">
+                    <option value="7">Last 7 Days</option>
+                    <option value="1">Last 24 Hours</option>
+                    <option value="30">Last 30 Days</option>
+                </select>
+                <div class="glass px-4 py-2 flex items-center text-sm font-semibold text-sky-400">
+                    <span class="w-2 h-2 bg-green-500 rounded-full mr-2"></span> Connected
+                </div>
+            </div>
+        </header>
+
+        <!-- Stats Grid -->
+        <div class="grid grid-cols-1 md:grid-cols-3 gap-6 mb-8">
+            <div class="glass p-6 glow">
+                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Trades</p>
+                <h3 id="statTotalTrades" class="text-4xl font-bold">--</h3>
+            </div>
+            <div class="glass p-6">
+                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Volume</p>
+                <h3 id="statTotalVolume" class="text-4xl font-bold">--</h3>
+            </div>
+            <div class="glass p-6">
+                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Tracked ISINs</p>
+                <h3 id="statIsins" class="text-4xl font-bold">--</h3>
+            </div>
+        </div>
+
+        <!-- Charts Container -->
+        <div class="grid grid-cols-1 xl:grid-cols-2 gap-8 mb-8">
+            <div class="glass p-6">
+                <h3 class="text-xl font-bold mb-6">Trade Volume Evolution</h3>
+                <div class="h-80"><canvas id="mainChart"></canvas></div>
+            </div>
+            <div class="glass p-6">
+                <h3 class="text-xl font-bold mb-6">Distribution by Continent</h3>
+                <div class="h-80"><canvas id="pieChart"></canvas></div>
+            </div>
+        </div>
+
+        <!-- Data Table -->
+        <div class="glass overflow-hidden">
+            <div class="p-6 border-b border-white/5 flex justify-between items-center">
+                <h3 class="text-xl font-bold">Company Metadata</h3>
+                <input type="text" placeholder="Search ISIN..." class="glass px-4 py-1 text-sm">
+            </div>
+            <table class="w-full text-left">
+                <thead class="bg-white/5 text-slate-400 text-sm uppercase">
+                    <tr>
+                        <th class="p-4 px-6">ISIN</th>
+                        <th class="p-4">Name</th>
+                        <th class="p-4">Country</th>
+                        <th class="p-4">Continent</th>
+                        <th class="p-4">Sector</th>
+                    </tr>
+                </thead>
+                <tbody id="metadataTable" class="divide-y divide-white/5">
+                    <!-- Rows injected via JS -->
+                </tbody>
+            </table>
+        </div>
+    </main>
+
+    <script>
+        const API_BASE = '/api';
+
+        async function fetchData() {
+            try {
+                const [tradesRes, metaRes, summaryRes] = await Promise.all([
+                    fetch(`${API_BASE}/trades?days=${document.getElementById('timeRange').value}`),
+                    fetch(`${API_BASE}/metadata`),
+                    fetch(`${API_BASE}/summary`)
+                ]);
+
+                const trades = await tradesRes.json();
+                const metadata = await metaRes.json();
+                const summary = await summaryRes.json();
+
+                updateStats(trades, metadata);
+                renderMainChart(trades);
+                renderPieChart(summary);
+                updateMetadataTable(metadata);
+            } catch (e) {
+                console.error("Dashboard error:", e);
+            }
+        }
+
+        function updateStats(trades, metadata) {
+            document.getElementById('statTotalTrades').innerText = trades.dataset ? trades.dataset.length.toLocaleString() : '0';
+            const totalVol = trades.dataset ? trades.dataset.reduce((acc, row) => acc + (row[4] * row[5]), 0) : 0;
+            document.getElementById('statTotalVolume').innerText = '€' + (totalVol/1000).toFixed(1) + 'k';
+            document.getElementById('statIsins').innerText = metadata.dataset ? metadata.dataset.length : '0';
+        }
+
+        let mainChart, pieChart;
+
+        function renderMainChart(trades) {
+            if (!trades.dataset) return;
+            const ctx = document.getElementById('mainChart').getContext('2d');
+            
+            // Basic aggregation for demonstration
+            const labels = trades.dataset.slice(-20).map(r => new Date(r[2]).toLocaleTimeString());
+            const data = trades.dataset.slice(-20).map(r => r[4]);
+
+            if (mainChart) mainChart.destroy();
+            mainChart = new Chart(ctx, {
+                type: 'line',
+                data: {
+                    labels,
+                    datasets: [{
+                        label: 'Price Evolution',
+                        data,
+                        borderColor: '#38bdf8',
+                        backgroundColor: 'rgba(56, 189, 248, 0.2)',
+                        fill: true,
+                        tension: 0.4,
+                        borderWidth: 3
+                    }]
+                },
+                options: {
+                    responsive: true,
+                    maintainAspectRatio: false,
+                    scales: {
+                        y: { grid: { color: 'rgba(255,255,255,0.05)' }, ticks: { color: '#94a3b8' } },
+                        x: { grid: { color: 'rgba(255,255,255,0.05)' }, ticks: { color: '#94a3b8' } }
+                    },
+                    plugins: { legend: { display: false } }
+                }
+            });
+        }
+
+        function renderPieChart(summary) {
+            if (!summary.dataset) return;
+            const ctx = document.getElementById('pieChart').getContext('2d');
+            const continents = summary.dataset.map(r => r[0]);
+            const volumes = summary.dataset.map(r => r[3]);
+
+            if (pieChart) pieChart.destroy();
+            pieChart = new Chart(ctx, {
+                type: 'doughnut',
+                data: {
+                    labels: continents,
+                    datasets: [{
+                        data: volumes,
+                        backgroundColor: ['#38bdf8', '#fbbf24', '#f87171', '#34d399', '#818cf8'],
+                        borderWidth: 0
+                    }]
+                },
+                options: {
+                    responsive: true,
+                    maintainAspectRatio: false,
+                    plugins: { legend: { position: 'bottom', labels: { color: '#94a3b8' } } }
+                }
+            });
+        }
+
+        function updateMetadataTable(metadata) {
+            const container = document.getElementById('metadataTable');
+            container.innerHTML = '';
+            if (!metadata.dataset) return;
+
+            metadata.dataset.forEach(row => {
+                const tr = document.createElement('tr');
+                tr.innerHTML = `
+                    <td class="p-4 px-6 font-mono text-sky-400">${row[0]}</td>
+                    <td class="p-4 font-semibold">${row[1]}</td>
+                    <td class="p-4"><span class="bg-white/10 px-2 py-1 rounded text-xs">${row[2]}</span></td>
+                    <td class="p-4 text-slate-400">${row[3]}</td>
+                    <td class="p-4 text-slate-400">${row[4]}</td>
+                `;
+                container.appendChild(tr);
+            });
+        }
+
+        document.getElementById('timeRange').addEventListener('change', fetchData);
+        fetchData();
+        setInterval(fetchData, 30000);
+    </script>
+</body>
+</html>
--- a/dashboard/server.py
+++ b/dashboard/server.py
@@ -0,0 +1,78 @@
+from fastapi import FastAPI, HTTPException, Depends
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+import requests
+import os
+import pandas as pd
+
+app = FastAPI(title="Trading Dashboard API")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Serve static files
+app.mount("/static", StaticFiles(directory="dashboard/public"), name="static")
+
+@app.get("/")
+async def read_index():
+    return FileResponse('dashboard/public/index.html')
+
+DB_USER = os.getenv("DB_USER", "admin")
+DB_PASSWORD = os.getenv("DB_PASSWORD", "quest")
+DB_AUTH = (DB_USER, DB_PASSWORD) if DB_USER and DB_PASSWORD else None
+DB_HOST = os.getenv("DB_HOST", "questdb")
+
+@app.get("/api/trades")
+async def get_trades(isin: str = None, days: int = 7):
+    query = f"select * from trades where timestamp > dateadd('d', -{days}, now())"
+    if isin:
+        query += f" and isin = '{isin}'"
+    query += " order by timestamp asc"
+    
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            return response.json()
+        throw_http_error(response)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/metadata")
+async def get_metadata():
+    query = "select * from metadata"
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            return response.json()
+        throw_http_error(response)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.get("/api/summary")
+async def get_summary():
+    # Group by continent/country if metadata exists
+    query = """
+    select m.continent, m.country, count(*) as trade_count, sum(t.price * t.quantity) as total_volume
+    from trades t
+    join metadata m on t.isin = m.isin
+    group by m.continent, m.country
+    """
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            return response.json()
+        throw_http_error(response)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+def throw_http_error(res):
+    raise HTTPException(status_code=res.status_code, detail=f"QuestDB error: {res.text}")
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,12 +15,13 @@ services:
      - QDB_HTTP_AUTH_ENABLED=true
      - QDB_HTTP_USER=${DB_USER:-admin}
      - QDB_HTTP_PASSWORD=${DB_PASSWORD:-quest}
-      # ILP Auth (optional, but good for consistency)
      - QDB_PG_USER=${DB_USER:-admin}
      - QDB_PG_PASSWORD=${DB_PASSWORD:-quest}

  fetcher:
-    build: .
+    build:
+      context: .
+      dockerfile: Dockerfile
    container_name: trading_fetcher
    depends_on:
      - questdb
@@ -30,5 +31,34 @@ services:
      - DB_USER=${DB_USER:-admin}
      - DB_PASSWORD=${DB_PASSWORD:-quest}

+  metadata_fetcher:
+    build:
+      context: .
+      dockerfile: Dockerfile.metadata
+    container_name: metadata_fetcher
+    depends_on:
+      - questdb
+    restart: always
+    environment:
+      - PYTHONUNBUFFERED=1
+      - DB_USER=${DB_USER:-admin}
+      - DB_PASSWORD=${DB_PASSWORD:-quest}
+      - DB_HOST=questdb
+
+  dashboard:
+    build:
+      context: .
+      dockerfile: Dockerfile.dashboard
+    container_name: trading_dashboard
+    ports:
+      - "8080:8000"
+    depends_on:
+      - questdb
+    restart: always
+    environment:
+      - DB_USER=${DB_USER:-admin}
+      - DB_PASSWORD=${DB_PASSWORD:-quest}
+      - DB_HOST=questdb
+
 volumes:
  questdb_data:
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,6 @@
 requests
 beautifulsoup4
+fastapi
+uvicorn
+pandas
+python-multipart
--- a/src/metadata/fetcher.py
+++ b/src/metadata/fetcher.py
@@ -0,0 +1,115 @@
+import requests
+import time
+import logging
+import os
+import datetime
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger = logging.getLogger("MetadataDaemon")
+
+DB_USER = os.getenv("DB_USER", "admin")
+DB_PASSWORD = os.getenv("DB_PASSWORD", "quest")
+DB_AUTH = (DB_USER, DB_PASSWORD) if DB_USER and DB_PASSWORD else None
+DB_HOST = os.getenv("DB_HOST", "questdb")
+
+def get_unique_isins():
+    query = "select distinct isin from trades"
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            data = response.json()
+            return [row[0] for row in data.get('dataset', []) if row[0]]
+    except Exception as e:
+        logger.error(f"Error fetching unique ISINs: {e}")
+    return []
+
+def get_processed_isins():
+    query = "select distinct isin from metadata"
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            data = response.json()
+            return [row[0] for row in data.get('dataset', []) if row[0]]
+    except Exception:
+        # Table might not exist yet
+        return []
+    return []
+
+def fetch_metadata(isin):
+    logger.info(f"Fetching metadata for ISIN: {isin}")
+    metadata = {
+        'isin': isin,
+        'name': 'Unknown',
+        'country': 'Unknown',
+        'continent': 'Unknown',
+        'sector': 'Unknown'
+    }
+    
+    # 1. GLEIF API for Name and Country
+    try:
+        gleif_url = f"https://api.gleif.org/api/v1/lei-records?filter[isin]={isin}"
+        res = requests.get(gleif_url, timeout=10)
+        if res.status_code == 200:
+            data = res.json().get('data', [])
+            if data:
+                attr = data[0].get('attributes', {})
+                metadata['name'] = attr.get('entity', {}).get('legalName', {}).get('name', 'Unknown')
+                metadata['country'] = attr.get('entity', {}).get('legalAddress', {}).get('country', 'Unknown')
+    except Exception as e:
+        logger.error(f"GLEIF error for {isin}: {e}")
+
+    # 2. Continent mapping from Country Code
+    if metadata['country'] != 'Unknown':
+        try:
+            country_url = f"https://restcountries.com/v3.1/alpha/{metadata['country']}"
+            res = requests.get(country_url, timeout=10)
+            if res.status_code == 200:
+                data = res.json()
+                if data and isinstance(data, list):
+                    continents = data[0].get('continents', [])
+                    if continents:
+                        metadata['continent'] = continents[0]
+        except Exception as e:
+            logger.error(f"RestCountries error for {metadata['country']}: {e}")
+
+    return metadata
+
+def save_metadata(metadata):
+    # QuestDB Influx Line Protocol
+    # table,tag1=val1 field1="str",field2=num timestamp
+    name = metadata['name'].replace(' ', '\\ ').replace(',', '\\,')
+    country = metadata['country']
+    continent = metadata['continent']
+    sector = metadata['sector']
+    isin = metadata['isin']
+    
+    line = f'metadata,isin={isin} name="{name}",country="{country}",continent="{continent}",sector="{sector}"'
+    
+    try:
+        response = requests.post(f"http://{DB_HOST}:9000/write", data=line + "\n", auth=DB_AUTH)
+        if response.status_code not in [200, 204]:
+            logger.error(f"Error saving metadata: {response.text}")
+    except Exception as e:
+        logger.error(f"Connection error to QuestDB: {e}")
+
+def main():
+    logger.info("Metadata Daemon started.")
+    while True:
+        unique_isins = get_unique_isins()
+        processed_isins = get_processed_isins()
+        
+        new_isins = [i for i in unique_isins if i not in processed_isins]
+        
+        if new_isins:
+            logger.info(f"Found {len(new_isins)} new ISINs to process.")
+            for isin in new_isins:
+                data = fetch_metadata(isin)
+                save_metadata(data)
+                time.sleep(1) # Rate limiting
+        else:
+            logger.info("No new ISINs found.")
+            
+        time.sleep(3600) # Run once per hour
+
+if __name__ == "__main__":
+    main()