fix: EIX scraping, LS endpoints, and premium Dashboard analytics

2026-01-23 18:09:00 +01:00
parent 3fc149599e
commit 1086c4aa1d
5 changed files with 232 additions and 79 deletions
--- a/.gitea/workflows/deploy.yaml
+++ b/.gitea/workflows/deploy.yaml
@@ -33,7 +33,7 @@ jobs:
            echo "DB_PASSWORD=${DB_PASSWORD}" >> .env
            
            # Docker Container neu bauen und starten
-            docker compose up -d --build
+            docker-compose up -d --build
            
            echo "Deployment abgeschlossen. Container laufen mit Passwortschutz."
        env:
--- a/dashboard/public/index.html
+++ b/dashboard/public/index.html
@@ -1,5 +1,6 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
@@ -13,44 +14,51 @@
            background-color: #0b0e14;
            color: #e2e8f0;
        }
+
        .glass {
            background: rgba(30, 41, 59, 0.5);
            backdrop-filter: blur(12px);
            border: 1px solid rgba(255, 255, 255, 0.1);
            border-radius: 16px;
        }
+
        .glow {
            box-shadow: 0 0 20px rgba(56, 189, 248, 0.2);
        }
+
        .sidebar-item:hover {
            background: rgba(56, 189, 248, 0.1);
            color: #38bdf8;
        }
+
        canvas {
            max-width: 100%;
        }
    </style>
 </head>
+
 <body class="flex min-h-screen">

    <!-- Sidebar -->
    <aside class="w-64 glass m-4 flex flex-col hidden lg:flex">
        <div class="p-6">
-            <h1 class="text-2xl font-bold bg-gradient-to-r from-sky-400 to-blue-500 bg-clip-text text-transparent">Antigravity Trade</h1>
+            <h1 class="text-2xl font-bold bg-gradient-to-r from-sky-400 to-blue-500 bg-clip-text text-transparent">
+                Antigravity Trade</h1>
        </div>
-        <nav class="flex-1 px-4 space-y-2 mt-4">
-            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition">
+        <nav class="flex-1 px-4 space-y-2 mt-4" id="sidebar">
+            <a href="#" onclick="showView('dashboard')"
+                class="sidebar-item flex items-center p-3 rounded-xl transition bg-sky-500/10 text-sky-400"
+                id="nav-dashboard">
                <span class="mr-3">📊</span> Dashboard
            </a>
-            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
-                <span class="mr-3">🌍</span> Global Markets
+            <a href="#" onclick="showView('analytics')" class="sidebar-item flex items-center p-3 rounded-xl transition"
+                id="nav-analytics">
+                <span class="mr-3">📈</span> Analytics
            </a>
-            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
+            <a href="#" onclick="showView('metadata')" class="sidebar-item flex items-center p-3 rounded-xl transition"
+                id="nav-metadata">
                <span class="mr-3">🏢</span> Companies
            </a>
-            <a href="#" class="sidebar-item flex items-center p-3 rounded-xl transition opacity-50">
-                <span class="mr-3">⚙️</span> Settings
-            </a>
        </nav>
        <div class="p-6 mt-auto">
            <div class="glass p-4 text-xs text-sky-400 border-sky-500/30">
@@ -78,94 +86,218 @@
            </div>
        </header>

-        <!-- Stats Grid -->
-        <div class="grid grid-cols-1 md:grid-cols-3 gap-6 mb-8">
-            <div class="glass p-6 glow">
-                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Trades</p>
-                <h3 id="statTotalTrades" class="text-4xl font-bold">--</h3>
+        <!-- View: Dashboard -->
+        <div id="content-dashboard" class="content-view">
+            <!-- Stats Grid -->
+            <div class="grid grid-cols-1 md:grid-cols-3 gap-6 mb-8">
+                <div class="glass p-6 glow">
+                    <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Trades</p>
+                    <h3 id="statTotalTrades" class="text-4xl font-bold">--</h3>
+                </div>
+                <div class="glass p-6">
+                    <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Volume</p>
+                    <h3 id="statTotalVolume" class="text-4xl font-bold">--</h3>
+                </div>
+                <div class="glass p-6">
+                    <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Tracked ISINs</p>
+                    <h3 id="statIsins" class="text-4xl font-bold">--</h3>
+                </div>
            </div>
-            <div class="glass p-6">
-                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Total Volume</p>
-                <h3 id="statTotalVolume" class="text-4xl font-bold">--</h3>
-            </div>
-            <div class="glass p-6">
-                <p class="text-slate-400 text-sm mb-1 text-uppercase tracking-wider">Tracked ISINs</p>
-                <h3 id="statIsins" class="text-4xl font-bold">--</h3>
+
+            <!-- Charts Container -->
+            <div class="grid grid-cols-1 xl:grid-cols-2 gap-8 mb-8">
+                <div class="glass p-6">
+                    <h3 class="text-xl font-bold mb-6">Price Evolution (Recent)</h3>
+                    <div class="h-80"><canvas id="mainChart"></canvas></div>
+                </div>
+                <div class="glass p-6">
+                    <h3 class="text-xl font-bold mb-6">Distribution by Continent</h3>
+                    <div class="h-80"><canvas id="pieChart"></canvas></div>
+                </div>
            </div>
        </div>

-        <!-- Charts Container -->
-        <div class="grid grid-cols-1 xl:grid-cols-2 gap-8 mb-8">
-            <div class="glass p-6">
-                <h3 class="text-xl font-bold mb-6">Trade Volume Evolution</h3>
-                <div class="h-80"><canvas id="mainChart"></canvas></div>
+        <!-- View: Analytics -->
+        <div id="content-analytics" class="content-view hidden">
+            <div class="glass p-6 mb-8">
+                <h3 class="text-xl font-bold mb-6">Exchange Comparison (Volume)</h3>
+                <div class="h-96"><canvas id="comparisonChart"></canvas></div>
            </div>
-            <div class="glass p-6">
-                <h3 class="text-xl font-bold mb-6">Distribution by Continent</h3>
-                <div class="h-80"><canvas id="pieChart"></canvas></div>
+            <div class="grid grid-cols-1 md:grid-cols-2 gap-8">
+                <div class="glass p-6">
+                    <h3 class="text-xl font-bold mb-6">EIX vs LS Trade Count</h3>
+                    <div class="h-64"><canvas id="tradeCountChart"></canvas></div>
+                </div>
+                <div class="glass p-6">
+                    <h3 class="text-xl font-bold mb-6">Average Trade Size</h3>
+                    <div id="avgTradeData" class="space-y-4">
+                        <!-- Dynamic data -->
+                    </div>
+                </div>
            </div>
        </div>

-        <!-- Data Table -->
-        <div class="glass overflow-hidden">
-            <div class="p-6 border-b border-white/5 flex justify-between items-center">
-                <h3 class="text-xl font-bold">Company Metadata</h3>
-                <input type="text" placeholder="Search ISIN..." class="glass px-4 py-1 text-sm">
+        <!-- View: Metadata (Table) -->
+        <div id="content-metadata" class="content-view hidden">
+
+            <!-- Data Table -->
+            <div class="glass overflow-hidden">
+                <div class="p-6 border-b border-white/5 flex justify-between items-center">
+                    <h3 class="text-xl font-bold">Company Metadata</h3>
+                    <input type="text" placeholder="Search ISIN..." class="glass px-4 py-1 text-sm">
+                </div>
+                <table class="w-full text-left">
+                    <thead class="bg-white/5 text-slate-400 text-sm uppercase">
+                        <tr>
+                            <th class="p-4 px-6">ISIN</th>
+                            <th class="p-4">Name</th>
+                            <th class="p-4">Country</th>
+                            <th class="p-4">Continent</th>
+                            <th class="p-4">Sector</th>
+                        </tr>
+                    </thead>
+                    <tbody id="metadataTable" class="divide-y divide-white/5">
+                        <!-- Rows injected via JS -->
+                    </tbody>
+                </table>
            </div>
-            <table class="w-full text-left">
-                <thead class="bg-white/5 text-slate-400 text-sm uppercase">
-                    <tr>
-                        <th class="p-4 px-6">ISIN</th>
-                        <th class="p-4">Name</th>
-                        <th class="p-4">Country</th>
-                        <th class="p-4">Continent</th>
-                        <th class="p-4">Sector</th>
-                    </tr>
-                </thead>
-                <tbody id="metadataTable" class="divide-y divide-white/5">
-                    <!-- Rows injected via JS -->
-                </tbody>
-            </table>
-        </div>
    </main>

    <script>
        const API_BASE = '/api';

+        function showView(viewId) {
+            document.querySelectorAll('.content-view').forEach(v => v.classList.add('hidden'));
+            document.getElementById(`content-${viewId}`).classList.remove('hidden');
+
+            document.querySelectorAll('#sidebar a').forEach(a => {
+                a.classList.remove('bg-sky-500/10', 'text-sky-400');
+            });
+            document.getElementById(`nav-${viewId}`).classList.add('bg-sky-500/10', 'text-sky-400');
+
+            const titles = {
+                'dashboard': ['Market Overview', 'Real-time trading analytics across multiple exchanges.'],
+                'analytics': ['Advanced Analytics', 'Deep dive comparison and historical trends.'],
+                'metadata': ['Enriched Company Data', 'Unified metadata from GLEIF and Financial APIs.']
+            };
+            document.querySelector('#viewTitle h2').innerText = titles[viewId][0];
+            document.querySelector('#viewTitle p').innerText = titles[viewId][1];
+            fetchData();
+        }
+
        async function fetchData() {
            try {
-                const [tradesRes, metaRes, summaryRes] = await Promise.all([
-                    fetch(`${API_BASE}/trades?days=${document.getElementById('timeRange').value}`),
+                const days = document.getElementById('timeRange').value;
+                const [tradesRes, metaRes, summaryRes, compRes] = await Promise.all([
+                    fetch(`${API_BASE}/trades?days=${days}`),
                    fetch(`${API_BASE}/metadata`),
-                    fetch(`${API_BASE}/summary`)
+                    fetch(`${API_BASE}/summary`),
+                    fetch(`${API_BASE}/comparison?days=${days}`)
                ]);

                const trades = await tradesRes.json();
                const metadata = await metaRes.json();
                const summary = await summaryRes.json();
+                const comparison = await compRes.json();

                updateStats(trades, metadata);
                renderMainChart(trades);
                renderPieChart(summary);
                updateMetadataTable(metadata);
+                renderComparisonCharts(comparison);
            } catch (e) {
                console.error("Dashboard error:", e);
            }
        }

        function updateStats(trades, metadata) {
-            document.getElementById('statTotalTrades').innerText = trades.dataset ? trades.dataset.length.toLocaleString() : '0';
-            const totalVol = trades.dataset ? trades.dataset.reduce((acc, row) => acc + (row[4] * row[5]), 0) : 0;
-            document.getElementById('statTotalVolume').innerText = '€' + (totalVol/1000).toFixed(1) + 'k';
-            document.getElementById('statIsins').innerText = metadata.dataset ? metadata.dataset.length : '0';
+            const rowCount = trades.dataset ? trades.dataset.length : 0;
+            document.getElementById('statTotalTrades').innerText = rowCount.toLocaleString();
+
+            let totalVol = 0;
+            if (trades.dataset) {
+                trades.dataset.forEach(row => {
+                    const p = parseFloat(row[4]) || 0;
+                    const q = parseFloat(row[5]) || 0;
+                    totalVol += (p * q);
+                });
+            }
+
+            if (totalVol >= 1000000) {
+                document.getElementById('statTotalVolume').innerText = '€' + (totalVol / 1000000).toFixed(2) + 'M';
+            } else {
+                document.getElementById('statTotalVolume').innerText = '€' + (totalVol / 1000).toFixed(1) + 'k';
+            }
+
+            const metaCount = metadata.dataset ? metadata.dataset.length : 0;
+            document.getElementById('statIsins').innerText = metaCount.toLocaleString();
        }

-        let mainChart, pieChart;
+        let mainChart, pieChart, compChart, countChart;
+
+        function renderComparisonCharts(data) {
+            if (!data.dataset) return;
+            const ctxComp = document.getElementById('comparisonChart').getContext('2d');
+            const ctxCount = document.getElementById('tradeCountChart').getContext('2d');
+
+            // Organize data: {exchange: {days: [], volumes: []}}
+            const exchanges = {};
+            const daysSet = new Set();
+
+            data.dataset.forEach(row => {
+                const ex = row[0];
+                const day = new Date(row[1]).toLocaleDateString();
+                const vol = row[2];
+                const count = row[3];
+
+                if (!exchanges[ex]) exchanges[ex] = { days: {}, volumes: {}, counts: {} };
+                exchanges[ex].volumes[day] = vol;
+                exchanges[ex].counts[day] = count;
+                daysSet.add(day);
+            });
+
+            const labels = Array.from(daysSet).sort((a, b) => new Date(a) - new Date(b));
+
+            const createDataset = (ex, type, color) => ({
+                label: ex,
+                data: labels.map(l => exchanges[ex][type][l] || 0),
+                borderColor: color,
+                backgroundColor: color + '33',
+                fill: true,
+                tension: 0.3
+            });
+
+            if (compChart) compChart.destroy();
+            compChart = new Chart(ctxComp, {
+                type: 'line',
+                data: {
+                    labels,
+                    datasets: [
+                        createDataset('LS', 'volumes', '#38bdf8'),
+                        createDataset('EIX', 'volumes', '#fbbf24')
+                    ]
+                },
+                options: { responsive: true, maintainAspectRatio: false }
+            });
+
+            if (countChart) countChart.destroy();
+            countChart = new Chart(ctxCount, {
+                type: 'bar',
+                data: {
+                    labels,
+                    datasets: [
+                        { label: 'LS', data: labels.map(l => exchanges['LS']?.counts[l] || 0), backgroundColor: '#38bdf8' },
+                        { label: 'EIX', data: labels.map(l => exchanges['EIX']?.counts[l] || 0), backgroundColor: '#fbbf24' }
+                    ]
+                },
+                options: { responsive: true, maintainAspectRatio: false }
+            });
+        }

        function renderMainChart(trades) {
            if (!trades.dataset) return;
            const ctx = document.getElementById('mainChart').getContext('2d');
-            
+
            // Basic aggregation for demonstration
            const labels = trades.dataset.slice(-20).map(r => new Date(r[2]).toLocaleTimeString());
            const data = trades.dataset.slice(-20).map(r => r[4]);
@@ -245,4 +377,5 @@
        setInterval(fetchData, 30000);
    </script>
 </body>
-</html>
+
+</html>
--- a/dashboard/server.py
+++ b/dashboard/server.py
@@ -59,7 +59,7 @@ async def get_summary():
    query = """
    select m.continent, m.country, count(*) as trade_count, sum(t.price * t.quantity) as total_volume
    from trades t
-    join metadata m on t.isin = m.isin
+    left join metadata m on t.isin = m.isin
    group by m.continent, m.country
    """
    try:
@@ -70,6 +70,24 @@ async def get_summary():
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

+@app.get("/api/comparison")
+async def get_comparison(days: int = 7):
+    # Aggregated volume per exchange per day
+    query = f"""
+    select exchange, date_trunc('day', timestamp) as day, sum(price * quantity) as daily_volume, count(*) as trade_count
+    from trades 
+    where timestamp > dateadd('d', -{days}, now())
+    group by exchange, day
+    order by day asc
+    """
+    try:
+        response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
+        if response.status_code == 200:
+            return response.json()
+        throw_http_error(response)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
 def throw_http_error(res):
    raise HTTPException(status_code=res.status_code, detail=f"QuestDB error: {res.text}")

--- a/src/exchanges/eix.py
+++ b/src/exchanges/eix.py
@@ -13,33 +13,34 @@ class EIXExchange(BaseExchange):
        return "EIX"

    def fetch_latest_trades(self, limit: int = 1) -> List[Trade]:
-        url = "https://european-investor-exchange.com/en/trade-list"
-        response = requests.get(url)
-        response.raise_for_status()
-        
-        soup = BeautifulSoup(response.text, 'html.parser')
-        next_data_script = soup.find('script', id='__NEXT_DATA__')
-        if not next_data_script:
+        # EIX stores its file list in a separate API endpoint
+        url = "https://european-investor-exchange.com/api/official-trades"
+        try:
+            response = requests.get(url, timeout=15)
+            response.raise_for_status()
+            files_list = response.json()
+        except Exception as e:
+            print(f"Error fetching EIX file list: {e}")
            return []
        
-        data = json.loads(next_data_script.string)
-        rows_data = data.get('props', {}).get('pageProps', {}).get('rowsData', [])
-        
        trades = []
        count = 0
-        for row in rows_data:
-            file_key = row.get('key')
+        for item in files_list:
+            file_key = item.get('fileName')
            if not file_key:
                continue
            
            # Download the CSV
            csv_url = f"https://european-investor-exchange.com/api/trade-file-contents?key={file_key}"
-            csv_response = requests.get(csv_url)
-            if csv_response.status_code == 200:
-                trades.extend(self._parse_csv(csv_response.text))
-                count += 1
-                if limit and count >= limit:
-                    break
+            try:
+                csv_response = requests.get(csv_url, timeout=20)
+                if csv_response.status_code == 200:
+                    trades.extend(self._parse_csv(csv_response.text))
+                    count += 1
+                    if limit and count >= limit:
+                        break
+            except Exception as e:
+                print(f"Error downloading EIX CSV {file_key}: {e}")
                
        return trades

--- a/src/exchanges/ls.py
+++ b/src/exchanges/ls.py
@@ -14,6 +14,7 @@ class LSExchange(BaseExchange):
        endpoints = ["https://www.ls-x.de/_rpc/json/.lstc/instrument/list/lstctradestoday"]
        if include_yesterday:
            endpoints.append("https://www.ls-x.de/_rpc/json/.lstc/instrument/list/lstctradesyesterday")
+            endpoints.append("https://www.ls-x.de/_rpc/json/.lstc/instrument/list/lsxtradesyesterday")
        
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',