完成日志、配置文件、数据库工具模块

api.py

-from framework import parse_date, get_quarter_start, get_config
 from abc import ABC, abstractmethod
 from enum import Enum, unique
-from dateutil.relativedelta import relativedelta
 
 
 @unique
@@ -158,9 +156,20 @@ class PortfoliosBuilder(ABC):
     @abstractmethod
     def get_portfolios(self, day, risk: PortfoliosRisk, type: PortfoliosType = PortfoliosType.NORMAL):
         '''
-        获取指定日期，指定风险等级的投资组合
+        获取指定日期，指定风险等级，指定类型的投资组合
+        :param type: 投组的类型
         :param day: 指定日期
         :param risk: 风险等级
         :return: 资产组合字典{id: weight}
         '''
         pass
+
+    @abstractmethod
+    def build_portfolio(self, day, type: PortfoliosType):
+        '''
+        构建指定日期，指定类型的投资组合
+        :param day: 指定日期
+        :param type: 指定类型
+        :return 投资组合数据{risk: {...}}，计算明细数据 {...}
+        '''
+        pass

asset_pool/asset_optimize.py

 import json
-import pandas as pd
 from abc import ABC, abstractmethod
+from datetime import datetime as dt
+
+import pandas as pd
 from dateutil.relativedelta import relativedelta
 from empyrical import sortino_ratio
-from framework import filter_weekend, dict_remove, get_config, component, autowired, get_quarter_start
+
 from api import AssetOptimize, Navs, BusinessException, Datum, AssetPoolType
 from asset_pool.dao import robo_assets_pool as rop
-from datetime import datetime as dt
+from framework import filter_weekend, dict_remove, get_config, component, autowired, get_quarter_start
 
 
 class SortinoAssetOptimize(AssetOptimize, ABC):
@@ -88,7 +90,8 @@ class FundSortinoAssetOptimize(SortinoAssetOptimize):
     def get_pct_change(self, fund_ids, day):
         if not self._config:
             raise BusinessException(f"find optimize, but not found sortino config.")
-        start = filter_weekend(sorted([day - relativedelta(days=1, **dict_remove(x, ('weight', 'name'))) for x in self._config])[0])
+        start = filter_weekend(
+            sorted([day - relativedelta(days=1, **dict_remove(x, ('weight', 'name'))) for x in self._config])[0])
         fund_navs = pd.DataFrame(self._navs.get_fund_navs(fund_ids=tuple(fund_ids), min_date=start, max_date=day))
         fund_navs.sort_values('nav_date', inplace=True)
         fund_navs = fund_navs.pivot_table(index='nav_date', columns='fund_id', values='nav_cal')

asset_pool/asset_pool.py

-from framework import component, autowired
-from api import AssetPool, AssetOptimize, AssetRisk
 from datetime import datetime as dt
 
+from api import AssetPool, AssetOptimize, AssetRisk
+from framework import component, autowired
+
 
 @component
 class FundAssetPool(AssetPool):

asset_pool/asset_risk.py

 import json
+import logging
 from datetime import datetime as dt
 
 import pandas as pd
@@ -7,7 +8,9 @@ from scipy.stats import norm
 
 from api import AssetRisk, Navs, AssetRiskDateType as DateType, Datum, AssetPoolType
 from asset_pool.dao import asset_risk_dates as ard, asset_ewma_value as aev, robo_assets_pool as rap
-from framework import component, autowired, get_config, log, format_date, block_execute
+from framework import component, autowired, get_config, format_date, block_execute
+
+logger = logging.getLogger(__name__)
 
 
 def get_risk_start_date():
@@ -38,7 +41,6 @@ class CvarEwmaAssetRisk(AssetRisk):
         return json.loads(asset_pool['asset_ids'])
 
     def is_risk(self, id, day) -> bool:
-        print(id, day)
         asset_pool = rap.get_one(day, AssetPoolType.RISK)
         if asset_pool:
             return id in json.loads(asset_pool['asset_ids'])
@@ -49,11 +51,11 @@ class CvarEwmaAssetRisk(AssetRisk):
     def build_risk_date(self, asset_id, day=dt.today()):
         risk_date = not None
         try:
-            log.debug(f"start build risk date for asset[{asset_id}] to date[{format_date(day)}]")
+            logger.info(f"start build risk date for asset[{asset_id}] to date[{format_date(day)}]")
             while risk_date is not None:
                 risk_date = self.get_next_date(asset_id, day=day)
         except Exception as e:
-            log.exception(f"build risk date for asset[{asset_id}] after date[{risk_date}] error", e)
+            logger.exception(f"build risk date for asset[{asset_id}] after date[{risk_date}] error", e)
 
     def get_next_date(self, asset_id, day=dt.today()):
         last = ard.get_last_one(asset_id, day)
@@ -83,7 +85,8 @@ class CvarEwmaAssetRisk(AssetRisk):
                 if row['rtn'] < rtns[rtns.date == cvar_start_date].iloc[0].rtn:
                     # 当日回报率跌破最低点, 则直接触发
                     tigger = True
-                elif row['rtn'] <= self._config['cvar']['threshold'] and len(cvar_rtns) >= self._config['cvar']['min-volume']:
+                elif row['rtn'] <= self._config['cvar']['threshold'] and len(cvar_rtns) >= self._config['cvar'][
+                    'min-volume']:
                     # 当日回报率小于等于阀值并且有足够cvar累计计算数据，则计算cvar判断
                     alpha = 1 - self._config['cvar']['coef']
                     mean = cvar_rtns['rtn'].mean()

asset_pool/dao/asset_ewma_value.py

 from framework import read, write, where, format_date
 
-
 __COLUMNS__ = {
     'aev_id': 'id',
     'aev_date': 'date',
@@ -30,7 +29,7 @@ def get_last_one(asset_id, max_date=None):
 
 @read
 def get_list(asset_id, min_date=None, max_date=None):
-    sqls =[]
+    sqls = []
     if min_date:
         sqls.append(f"aev_date >= '{format_date(min_date)}'")
     if max_date:

asset_pool/dao/asset_risk_dates.py

-from framework import read, write, where, format_date
 from api import AssetRiskDateType as DateType
+from framework import read, write, where, format_date
 
 __COLUMNS__ = {
     'ard_id': 'id',
@@ -28,4 +28,3 @@ def get_last_one(fund_id, date, type: DateType = None):
     select {','.join([f"`{x[0]}` as `{x[1]}`" for x in __COLUMNS__.items()])} 
     from asset_risk_dates {where(sql, **kwargs)} order by ard_date desc, ard_type asc limit 1
     '''
-

asset_pool/dao/robo_assets_pool.py

 import json
-from framework import read, write, where, format_date
+
 from api import AssetPoolType
+from framework import read, write, where, format_date
 
 __COLUMNS__ = {
     'rap_id': 'id',

basic/dao/robo_fund_navs.py

 from framework import read, where, format_date, to_tuple
 
-
 __COLUMNS__ = {
     'rfn_fund_id': 'fund_id',
     'rfn_date': 'nav_date',

basic/datum.py

 import json
+
 from api import DatumType, Datum
 from basic.dao import robo_base_datum as rbd
 from framework import component, parse_date

basic/navs.py

 import pandas as pd
+
 from api import Navs, Datum
 from basic.dao import robo_exrate as re, robo_fund_navs as rfn
 from framework import get_config, component, autowired
@@ -18,7 +19,8 @@ class DefaultNavs(Navs):
             navs = pd.DataFrame(navs)
             navs = navs.pivot_table(index='nav_date', columns='fund_id', values='nav_cal')
             for exrate_config in self._config['exrate']:
-                exrate = pd.DataFrame(re.get_exrates(ticker=exrate_config['ticker'], min_date=navs.index.min(), max_date=navs.index.max()))
+                exrate = pd.DataFrame(re.get_exrates(ticker=exrate_config['ticker'], min_date=navs.index.min(),
+                                                     max_date=navs.index.max()))
                 exrate = exrate[['date', 'close']]
                 exrate.set_index('date', inplace=True)
                 for fund in self._datum.get_fund_datums(crncy=exrate_config['from']):
@@ -30,7 +32,3 @@ class DefaultNavs(Navs):
             navs.sort_values(by=['fund_id', 'nav_date'], inplace=True)
             navs = navs.to_dict('records')
         return navs
-
-
-if __name__ == '__main__':
-    print(isinstance((), tuple))

config.yml

@@ -16,10 +16,10 @@ framework:
     user: jft-ra@thizgroup.com
     password: 5dbb#30ec6d3
   mulit-process:
-    max-workers: 4
+    max-workers: 8
   logger:
     version: 1
-    use: ${LOG_NAME:root}
+    use: prod
     formatters:
       brief:
         format: "%(asctime)s - %(levelname)s - %(message)s"
@@ -29,7 +29,7 @@ framework:
       console:
         class: logging.StreamHandler
         formatter: simple
-        level: INFO
+        level: DEBUG
         stream: ext://sys.stdout
       file:
         class: logging.handlers.TimedRotatingFileHandler
@@ -42,9 +42,11 @@ framework:
         when: D
     loggers:
       prod:
-        handlers: [ console,file ]
+        handlers: [ console, file ]
         level: INFO
-        propagate: 0
+        propagate: no
+      portfolios:
+        level: DEBUG
     root:
       level: INFO
       handlers: [ console ]

framework/__init__.py

@@ -2,9 +2,9 @@ from .date_utils import *
 from .base import *
 from .database import read, write, transaction, where, to_columns
 from .env_config import config, get_config
-from .logger import build_logger, logger as log
+from .logs import build_logger, get_logger
 from .injectable import component, autowired, get_instance, init_injectable as _init_injectable
 from .mulit_process import process_pool, create_process_pool, block_execute
 
 _init_injectable()
-del injectable, logger, env_config, database, base, date_utils, _init_injectable, mulit_process
+del injectable, logs, env_config, database, base, date_utils, _init_injectable, mulit_process

framework/env_config.py

@@ -4,7 +4,7 @@ from functools import partial
 
 import yaml
 
-from .base import *
+from framework.base import *
 
 has_regex_module = False
 ENV_VAR_MATCHER = re.compile(
@@ -17,7 +17,6 @@ ENV_VAR_MATCHER = re.compile(
     """, re.VERBOSE
 )
 
-
 IMPLICIT_ENV_VAR_MATCHER = re.compile(
     r"""
         .*          # matches any number of any characters
@@ -27,7 +26,6 @@ IMPLICIT_ENV_VAR_MATCHER = re.compile(
     """, re.VERBOSE
 )
 
-
 RECURSIVE_ENV_VAR_MATCHER = re.compile(
     r"""
         \$\{       # match characters `${` literally
@@ -89,7 +87,9 @@ yaml.add_implicit_resolver(
 config = build_config()
 
 
-def get_config(module: str = None):
+def get_config(module: str = None, file: str = None):
+    if module == '__main__':
+        module = file
     result = config
     if module:
         for name in [x.replace('_', '-') for x in module.split('.')]:

framework/logger.py → framework/logs.py

+import logging
 import os
-from logging import config as cf, getLogger
-from framework.env_config import get_config
+from logging import config as cf
+
 from framework.base import get_project_path
+from framework.env_config import get_config
 
 
-def build_logger(config, name='root'):
+def build_logger(config):
     if 'handlers' in config and 'file' in config['handlers']:
         file = config['handlers']['file']
         path = os.path.join(get_project_path(), file["filename"])
         os.makedirs(os.path.split(path)[0], exist_ok=True)
         file["filename"] = os.path.abspath(path)
     cf.dictConfig(config)
-    return getLogger(name)
 
 
-config = get_config(__name__)
-logger = build_logger(config, name=config['use'] if 'use' in config else None) if config else None
+config = get_config("framework.logger")
+if config:
+    build_logger(config)
+
+
+def get_logger(name=None):
+    return logging.getLogger(config['use'] if 'use' in config and config['use'] is not None else name)

framework/mulit_process.py

 from concurrent.futures import ProcessPoolExecutor, as_completed
+
 from framework.env_config import get_config
-from functools import partial, wraps
 
 config = get_config(__name__)
 process_pool = ProcessPoolExecutor(max_workers=config['max-workers'] or 2)

main.py

-from framework import autowired, parse_date, log
+from framework import autowired, parse_date, get_logger
 from api import PortfoliosBuilder, PortfoliosRisk
 
+logger = get_logger('main')
 
-@autowired
+
+@autowired(names={'builder': 'poem'})
 def start(builder: PortfoliosBuilder = None):
     day = parse_date('2022-11-07')
-    log.info(builder.get_portfolios(day, PortfoliosRisk.FT3))
-
-
-def test(arg):
-    if arg:
-        return 1, 1
-    else:
-        return None
+    logger.info(builder.get_portfolios(day, PortfoliosRisk.FT3))
 
 
 if __name__ == '__main__':
-    log.info("start")
-    start()
\ No newline at end of file
+    logger.info("info")
+    logger.debug('debug')
+    logger.warning('warning')
+    logger.error('error')
+    logger.critical('critical')
+    # start()

portfolios/dao/robo_mpt_portfolios.py

 from datetime import datetime
-from framework import read, write, where, format_date
 from enum import Enum
+
 from api import PortfoliosRisk, PortfoliosType
+from framework import read, write, where, format_date
 
 __COLUMNS__ = {
     'rmp_id': 'id',
@@ -31,6 +32,6 @@ def insert(datas):
 @read(one=True)
 def get_one(day, type: PortfoliosType, risk: PortfoliosRisk):
     return f'''
-    select {','.join([x for x in __COLUMNS__.keys()])} from robo_mpt_portfolios
+    select {','.join([f"{x[0]} as {x[1]}" for x in __COLUMNS__.items()])} from robo_mpt_portfolios
     {where(rmp_date=day, rmp_risk=risk, rmp_type=type)} 
     '''

test.py

+from api import PortfoliosBuilder, PortfoliosType
+from framework import autowired, parse_date, get_logger
+
+logger = get_logger('test')
+
+
+@autowired(names={'builder': 'poem'})
+def test_portfolio_builder(builder: PortfoliosBuilder = None):
+    result, detail = builder.build_portfolio(parse_date('2011-11-07'), PortfoliosType.NORMAL)
+    logger.info(result)
+    logger.info(detail)
+
+
+if __name__ == '__main__':
+    test_portfolio_builder()