增加彭博基金抓取逻辑

basic/sync.py

@@ -54,7 +54,7 @@ class JDCDataSync(DataSync, ABC):
             while True:
                 url = self.build_urls(datum=datum, page=page, start_date=start_date)
                 if url is None:
-                    raise Exception(f'''request data {datum['id']} not exist!''')
+                    break
                 response = requests.get(url).json()
                 if not response['success']:
                     raise Exception(f'''request indictor failed: {response['status']}''')
@@ -177,7 +177,7 @@ class IndexSync(JDCDataSync):
         return next_workday(last['date']) if last else self.start_date
 
     def build_urls(self, datum, start_date, page=0) -> str:
-        return f'http://jdcprod.thiztech.com/api/datas/index-value?page={page}&size=200&sourceCode={quote(datum["bloombergTicker"])}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
+        return f'https://jdcprod.thiztech.com/api/datas/index-value?page={page}&size=200&sourceCode={quote(datum["bloombergTicker"])}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
 
     def store_date(self, datumid, datas: List[dict]):
         save_datas = [{
@@ -211,7 +211,7 @@ class EcoSync(JDCDataSync):
         return next_workday(last['date']) if last else self.start_date
 
     def build_urls(self, datum, start_date, page=0) -> str:
-        return f'http://jdcprod.thiztech.com/api/datas/eco-value?page={page}&size=200&sourceCode={quote(datum["bloombergTicker"])}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
+        return f'https://jdcprod.thiztech.com/api/datas/eco-value?page={page}&size=200&sourceCode={quote(datum["bloombergTicker"])}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
 
     def store_date(self, datumid, datas: List[dict]):
         save_datas = [{
@@ -229,7 +229,7 @@ class FundNavSync(JDCDataSync):
 
     def __init__(self):
         super(FundNavSync, self).__init__()
-        self._subject_keys = self.find_jdc_subject_key()
+        self._jdc_querys = self.find_jdc_querys()
 
     @property
     def datum_type(self) -> DatumType:
@@ -240,20 +240,31 @@ class FundNavSync(JDCDataSync):
         return next_workday(last['nav_date']) if last else self.start_date
 
     def build_urls(self, datum, start_date, page=0) -> str:
-        if datum['id'] not in self._subject_keys:
+        if datum['id'] not in self._jdc_querys:
             return None
-        key = self._subject_keys[datum['id']]
-        return f'http://jdcprod.thiztech.com/api/datas/asset-value?subjectKeys={key}&page={page}&size=200&sourceType=TW&startDate={format_date(start_date)}'
-
-    def find_jdc_subject_key(self):
-        funds = {x['isin']: x for x in self._datum.get_datums(type=DatumType.FUND)}
-        response = requests.get('http://jdcprod.thiztech.com/api/subject?busiField=TW&sourceType=TW&subjectType=FUND')
+        querys = self._jdc_querys[datum['id']]
+        query_str = '&'.join([f'{x[0]}={quote(str(x[1]).encode())}' for x in querys.items()])
+        return f'https://jdcprod.thiztech.com/api/datas/asset-value?page={page}&size=200&startDate={format_date(start_date)}&{query_str}'
+
+    def find_jdc_querys(self):
+        funds = self._datum.get_datums(type=DatumType.FUND, exclude=False)
+        urls = {x['id']: {
+            'sourceCode': x['bloombergTicker'],
+            'sourceType': 'BLOOMBERG'
+        } for x in funds if 'ftTicker' not in x and 'bloombergTicker' in x}
+
+        ft_tickers = {x['ftTicker']: x for x in funds if 'ftTicker' in x}
+        response = requests.get('https://jdcprod.thiztech.com/api/subject?busiField=TW&sourceType=TW&subjectType=FUND')
         response = response.json()
         if not response['success']:
             raise CollectError(f'''find fund subject failed: {response['status']}''')
-        content = response['body']['content']
-        content = [x for x in content if x.get('isin')]
-        return {funds[x['isin']]['id']: x['key'] for x in content if x['isin'] in funds}
+        return {**urls, **{
+            ft_tickers[x['fundId']]['id']: {
+                'subjectKeys': x['key'],
+                'sourceType': 'TW'
+            } for x in response['body']['content'] if x['fundId'] in ft_tickers
+        }}
+
 
     def store_date(self, datumid, datas: List[dict]):
         save_navs = [{
@@ -287,7 +298,7 @@ class ExrateSync(DataSync):
         return [x['ticker'] for x in navs_config['exrate']] if 'exrate' in navs_config else []
 
     def build_url(self, ticker, start_date, page=0):
-        return f'http://jdcprod.thiztech.com/api/datas/exrate-value?page={page}&size=200&sourceCode={quote(ticker)}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
+        return f'https://jdcprod.thiztech.com/api/datas/exrate-value?page={page}&size=200&sourceCode={quote(ticker)}&sourceType=BLOOMBERG&startDate={format_date(start_date)}'
 
     def do_sync(self, max_date=dt.today()):
         logger.info(f'start sync datas for type[EXRATE]')