-
-
+
+
diff --git a/dashboard/server.py b/dashboard/server.py
index 6c3e4ee..e76b527 100644
--- a/dashboard/server.py
+++ b/dashboard/server.py
@@ -55,12 +55,15 @@ async def get_metadata():
@app.get("/api/summary")
async def get_summary():
- # Group by continent/country if metadata exists
+ # Coalesce null values to 'Unknown' and group properly
query = """
- select m.continent, m.country, count(*) as trade_count, sum(t.price * t.quantity) as total_volume
+ select
+ coalesce(m.continent, 'Unknown') as continent,
+ count(*) as trade_count,
+ sum(t.price * t.quantity) as total_volume
from trades t
left join metadata m on t.isin = m.isin
- group by m.continent, m.country
+ group by continent
"""
try:
response = requests.get(f"http://{DB_HOST}:9000/exec", params={'query': query}, auth=DB_AUTH)
@@ -73,8 +76,14 @@ async def get_summary():
@app.get("/api/comparison")
async def get_comparison(days: int = 7):
# Aggregated volume per exchange per day
+ # QuestDB date_trunc('day', timestamp) is good.
+ # We ensure we get exchange, day, and metrics.
query = f"""
- select exchange, date_trunc('day', timestamp) as day, sum(price * quantity) as daily_volume, count(*) as trade_count
+ select
+ exchange,
+ date_trunc('day', timestamp) as day,
+ sum(price * quantity) as daily_volume,
+ count(*) as trade_count
from trades
where timestamp > dateadd('d', -{days}, now())
group by exchange, day
diff --git a/src/metadata/fetcher.py b/src/metadata/fetcher.py
index 906e3d6..0258602 100644
--- a/src/metadata/fetcher.py
+++ b/src/metadata/fetcher.py
@@ -35,6 +35,8 @@ def get_processed_isins():
return []
return []
+from bs4 import BeautifulSoup
+
def fetch_metadata(isin):
logger.info(f"Fetching metadata for ISIN: {isin}")
metadata = {
@@ -45,10 +47,15 @@ def fetch_metadata(isin):
'sector': 'Unknown'
}
+ # Common headers to avoid blocks
+ headers = {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+ }
+
# 1. GLEIF API for Name and Country
try:
gleif_url = f"https://api.gleif.org/api/v1/lei-records?filter[isin]={isin}"
- res = requests.get(gleif_url, timeout=10)
+ res = requests.get(gleif_url, headers=headers, timeout=10)
if res.status_code == 200:
data = res.json().get('data', [])
if data:
@@ -58,11 +65,25 @@ def fetch_metadata(isin):
except Exception as e:
logger.error(f"GLEIF error for {isin}: {e}")
- # 2. Continent mapping from Country Code
+ # 2. Yahoo Finance for Sector
+ try:
+ # We use the lookup URL as discussed
+ yahoo_url = f"https://finance.yahoo.com/lookup/?s={isin}"
+ res = requests.get(yahoo_url, headers=headers, timeout=10)
+ if res.status_code == 200:
+ soup = BeautifulSoup(res.text, 'html.parser')
+ # Look for the sector link in the results table
+ sector_link = soup.find('a', href=lambda x: x and '/sector/' in x)
+ if sector_link:
+ metadata['sector'] = sector_link.text.strip()
+ except Exception as e:
+ logger.error(f"Yahoo sector error for {isin}: {e}")
+
+ # 3. Continent mapping from Country Code
if metadata['country'] != 'Unknown':
try:
country_url = f"https://restcountries.com/v3.1/alpha/{metadata['country']}"
- res = requests.get(country_url, timeout=10)
+ res = requests.get(country_url, headers=headers, timeout=10)
if res.status_code == 200:
data = res.json()
if data and isinstance(data, list):