新增投组校验，防止投组全为ft的情况

ai/EstimateMarketTrendV20.py

@@ -2,6 +2,7 @@ from typing import List
 
 from py_jftech import autowired
 
+from ai.dao.robo_datas import get_base_info
 from ai.data_access import DataAccess
 from ai.model_trainer import ModelTrainer
 from ai.noticer import upload_predict
@@ -27,15 +28,17 @@ def sync(syncs: List[DataSync] = None):
         s.do_sync()
 
 
-def predictionFromMoel(the_model, scaledX_forecast, predict_item):
+def predictionFromMoel(the_model, scaledX_forecast, predict_item, indexDict: dict):
     prediction = the_model.predict(scaledX_forecast)
     predictionStr = 'DOWN'
     if (prediction > 0.5):
         predictionStr = 'UP'
     content = f"""\n On day {forecastDay.strftime("%m/%d/%Y")}, the model predicts {predict_item} to be {predictionStr} in {str(numForecastDays)} business days. \n"""
     print(content)
-    # if predict_item == 'SPX':
-    #     upload_predict(f'{predict_item} Index', forecastDay, predictionStr)
+    # 上传预测结果
+    # key = [k for k, v in indexDict.items() if v == predict_item]
+    # index_info = get_base_info(key)[0]
+    # upload_predict(index_info['ticker'], forecastDay, predictionStr)
     # send(content)
     return prediction
 
@@ -113,4 +116,4 @@ if __name__ == '__main__':
         ensemble_model = trainer.ensemble_model(rf_model, gbt_model, svc_model, X_train, y_train, X_test, y_test)
 
         if (toForecast):
-            predictionFromMoel(ensemble_model, scaledX_forecast, indexDict[pid])
+            predictionFromMoel(ensemble_model, scaledX_forecast, indexDict[pid], indexDict)

api.py

@@ -227,10 +227,10 @@ class AssetOptimize(ABC):
     '''
 
     @abstractmethod
-    def find_optimize(self, ids, day):
+    def find_optimize(self, fund_ids, day):
         '''
         从多id中，选出指定日期最优的id
-        :param ids: 待选id列表
+        :param fund_ids: 待选id列表
         :param day: 指定日期
         :return: 最优的id
         '''
@@ -305,6 +305,22 @@ class PortfoliosBuilder(ABC):
         pass
 
 
+class PortfoliosChecker(ABC):
+    '''
+    投组组合检测器
+    '''
+
+    @abstractmethod
+    def check(self, day=None, portfolios=None):
+        """
+        检测避免出现最优投组同时出现全部是ft或美盛基金的情况，增加一步替换动作。
+        @param day:
+        @param portfolios:
+        @return:
+        """
+        pass
+
+
 class Solver(ABC):
     '''
     解算器

asset_pool/asset_optimize.py

@@ -110,7 +110,7 @@ class FundDividendSortinoAssetOptimize(SortinoAssetOptimize):
         sortino['score'] = sortino.apply(lambda r: sum([x['weight'] * r[x['name']] for x in self._config]), axis=1)
         sortino.sort_values('score', ascending=False, inplace=True)
         # 取得分数高的前optimize_count个
-        return pct_change.columns[sortino.index[0:self.optimize_count]].values
+        return pct_change.columns[sortino.index[0:self.optimize_count]].values,sortino['score']
 
     def get_optimize_pool(self, day):
         opt_pool = rop.get_one(day=day, type=AssetPoolType.OPTIMIZE)
@@ -125,7 +125,7 @@ class FundDividendSortinoAssetOptimize(SortinoAssetOptimize):
             for fund_group in self.get_groups():
                 fund_group = [x for x in fund_group if min_dates[x] <= max_incept_date]
                 if len(fund_group) > self.optimize_count:
-                    pool.extend(self.find_optimize(tuple(fund_group), day))
+                    pool.extend(self.find_optimize(tuple(fund_group), day)[0])
                 elif len(fund_group) <= self.optimize_count:
                     pool.extend(fund_group)
             rop.insert(day, AssetPoolType.OPTIMIZE, sorted(pool))

config-svrobo4.yml

@@ -107,6 +107,9 @@ portfolios: # 投组模块
 #      high-weight: [ 1 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
     poem: # poem相关
       cvar-scale-factor: 0.1 # 计算时用到的系数
+  checker: #投组检测模块
+    switch: off  #是否开启检查
+    custom-type-priority: [ 3,2,1,4 ]  # 检测优先级
 reports: # 报告模块相关
   navs:
     type: FUND

config-svrobo5.yml

@@ -112,6 +112,9 @@ portfolios: # 投组模块
 #      high-weight: [ 1 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
     poem: # poem相关
       cvar-scale-factor: 0.1 # 计算时用到的系数
+  checker: #投组检测模块
+    switch: off  #是否开启检查
+    custom-type-priority: [ 3,2,1,4 ]  # 检测优先级
 reports: # 报告模块相关
   navs:
     type: FUND

config-svrobo6.yml

@@ -111,6 +111,9 @@ portfolios: # 投组模块
       high-weight: [ 0.35 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
     poem: # poem相关
       cvar-scale-factor: 0.1 # 计算时用到的系数
+  checker: #投组检测模块
+    switch: on  #是否开启检查
+    custom-type-priority: [ 3,2,1,4 ]  # 检测优先级
 reports: # 报告模块相关
   navs:
     type: FUND

config-svrobo_Mdiv_PRR3.yml

+py-jftech:
+  logger:
+    version: 1
+    formatters:
+      brief:
+        format: "%(asctime)s - %(levelname)s - %(message)s"
+      simple:
+        format: "%(asctime)s - %(filename)s - %(levelname)s - %(message)s"
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        level: DEBUG
+        stream: ext://sys.stdout
+      file:
+        class: logging.handlers.TimedRotatingFileHandler
+        level: INFO
+        formatter: brief
+        filename: ${LOG_FILE:logs/info.log}
+        interval: 1
+        backupCount: 30
+        encoding: utf8
+        when: D
+#    loggers:
+#      basic.sync:
+#        level: DEBUG
+#        handlers: [console]
+#        propagate: no
+    root:
+      level: ${LOG_LEVEL:INFO}
+      handlers: ${LOG_HANDLERS:[ console ]}
+  database:
+    host: ${MYSQL_HOST:192.168.68.85}
+    port: ${MYSQL_PORT:3306}
+    user: ${MYSQL_USER:root}
+    password: ${MYSQL_PWD:changeit}
+    dbname: ${MYSQL_DBNAME:mdiv_prr3}  # mdiv_prr3
+  injectable:
+    names:
+      backtest: robo_executor.BacktestExecutor
+      datum: basic.datum.DefaultDatum
+      hold-report: portfolios.holder.DivHoldReportor
+      mpt: portfolios.builder.PoemARCPortfoliosBuilder
+      dividend-holder: portfolios.holder.InvTrustPortfoliosHolder
+      navs-sync: basic.sync.FundNavSync
+  email:
+    server: smtphz.qiye.163.com
+    user: jft-ra@thizgroup.com
+    password: 5dbb#30ec6d3
+  mulit-process:
+    max-workers: ${MAX_PROCESS:1}
+basic: # 基础信息模块
+  sync:
+    start-date: 1990-01-01 # 同步数据开始日期
+  datum: # 资料模块
+    change:
+      date: ${DATUM_CHANGE_DATE}
+      file: ${DATUM_CHANGE_FILE}
+    excludes: # 排除的资料彭博ticker
+#      backtest:
+#        - 'TEMDLRI LX Equity'  # 富蘭克林坦伯頓全球投資系列-美元短期票券基金 美元A(Mdis)股
+#        - 'LMWADMU ID Equity'  # 美盛西方資產全球藍籌債券基金 A 美元 配息
+#        - 'TGEIADI LX Equity'  # 富蘭克林坦伯頓全球投資系列-全球股票收益基金 美元A穩定月配息股
+      real:
+        - 'FGFSACU LX Equity'
+  navs: # 净值模块
+    exrate: # 汇率，如果不开启，整个这块注释掉
+      - from: EUR # 需要转换的货币类型
+        ticker: EURUSD BGN Curncy # 汇率值的彭博ticker
+asset-pool: # 资产池模块
+  asset-optimize: # 资产优选模块
+    sortino-weight: # sortino计算需要的权重，下面每一条为一次计算，e.g. months: 3, weight: 0.5 表示 3个月数据使用权重0.5来计算分值
+      - months: 3
+        weight: 0.5
+      - months: 6
+        weight: 0.3
+      - years: 1
+        weight: 0.2
+    asset-include: {'customType':[1,2,3,4]}
+    optimize-count: 3 #基金优选个数
+portfolios: # 投组模块
+  holder: # 持仓投组相关
+    init-nav: 100 # 初始金额
+    min-interval-days: 10 # 两次实际调仓最小间隔期，单位交易日
+    dividend-rate: 0.0 #设定年化配息率
+    dividend-date: 15 #配息日，每月15号
+    dividend-adjust-day: [1,4,7,10]  #每年的首个季度调整配息
+    warehouse-frequency: 1  #每隔1个月调一次仓
+    warehouse-transfer-date: 1  #调仓日
+    redeem-list: [ 'TEUSAAU LX Equity', 'LIGTRAA ID Equity', 'TEMFHAC LX Equity', 'LUSHUAA ID Equity' ] #从持仓中的低风险资产“直接”按序赎回
+  solver: # 解算器相关
+    model: prr # 结算模型 ARC ,PRR,  ~ 标准解算器
+    arc: on  #是否开启ARC
+    brr: 0.02 #误差补偿值
+    trr: 3
+    tol: 1E-10 # 误差满足条件
+    navs: # 净值要求
+      range: # 需要净值数据的区间, days: 90 表示90自然日，months: 3 表示3个自然月
+        days: 90
+      max-nan: # 最大缺失净值条件
+        asset: 8 # 单一资产最多缺少多少交易日数据，则踢出资产池
+        day: 0.5 # 单一交易日最多缺少百分之多少净值，则删除该交易日
+      risk: []  # 资产风险等级要求，可分开写也可以合并写，e.g. risk：[ 2, 3 ] 则表示 所有投组资产风险等级都是 2 或 3
+    LARC: [0.30, 0.00, 0.00]  #低阈值
+    UARC: [0.70, 0.70, 0.70] #高阈值
+    matrix-rtn-days: 20 # 计算回报率矩阵时，回报率滚动天数
+    asset-count: [5,5] # 投组资产个数。e.g. count 或 [min, max] 分别表示 最大最小都为count 或 最小为min 最大为max，另外这里也可以类似上面给不同风险等级分别配置
+    mpt: # mpt计算相关
+      cvar-beta: 0.2 # 计算Kbeta 需要用到
+      quantile: 0.9 # 分位点，也可以给不同风险等级分别配置
+      low-weight: 0.05 # 最低权重
+      high-weight: [ 0.35 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
+    poem: # poem相关
+      cvar-scale-factor: 0.1 # 计算时用到的系数
+  checker:  #投组检测模块
+    switch: on  #是否开启检查
+    custom-type-priority: [3,2,1,4]  # 检测优先级
+
+reports: # 报告模块相关
+  navs:
+    type: FUND
+    tickers:
+      - TEMTECI LX Equity
+      - TEPLX US Equity
+      - FRDPX US Equity
+      - FKRCX US Equity
+      - FTNRACU LX Equity
+  benchmark: # benchmark报告
+    ft:
+      init-amount: 100 # 初始金额
+      stock-rate: # stock型基金比例
+        RR3: 0.3
+        RR4: 0.5
+        RR5: 0.7
+  fixed-range: # 固定区间收益率
+    range-dates: # 固定起始截止日期
+      - start: 2008-01-01
+        end: 2008-10-27
+      - start: 2011-05-02
+        end: 2011-10-04
+      - start: 2013-05-08
+        end: 2013-06-24
+      - start: 2014-09-03
+        end: 2014-12-16
+      - start: 2015-04-28
+        end: 2016-01-21
+      - start: 2018-01-26
+        end: 2018-10-29
+      - start: 2020-01-20
+        end: 2020-03-23
+  relative-range: # 相对区间收益率
+    range-dates: # 相对时间周期
+      - days: 1
+        name: '一天'
+      - weeks: 1
+        name: '一周'
+      - months: 1
+        name: '一月'
+      - months: 3
+        name: '三月'
+      - months: 6
+        name: '六月'
+      - years: 1
+        name: '一年'
+      - years: 2
+        name: '两年'
+      - years: 3
+        name: '三年'
+      - years: 5
+        name: '五年'
+      - years: 10
+        name: '十年'
+      - dates: ~
+        name: '成立以来'
+  exports:
+    backtest: # 回测导出曹策略
+      save-path: ${EXPORT_PATH:excels} # 导出报告文件存放路径，如果以./或者../开头，则会以执行python文件为根目录，如果以/开头，则为系统绝对路径，否则，以项目目录为根目录
+      file-name: ${EXPORT_FILENAME:real} # 导出报告的文件名
+      save-config: ${EXPORT_CONFIG:off} # 是否保存配置文件
+      include-report: # 需要导出的报告类型列表，下面的顺序，也代表了excel中sheet的顺序
+  #      - funds-report # 基金资料
+  #      - navs-report # 净值报告
+        - hold-report # 持仓报告
+        - signal-report # 信号报告
+        - benckmark-report # benckmark报告
+        - combo-report # 持仓对比
+        - indicators-report # 各种特殊指标报告
+        - fixed-range-report # 固定区间收益报告
+        - relative-range-report # 相对区间收益报告
+        - year-range-report # 单年区间业绩报告
+        - month-div-rate-report # 月度配息率比较
+        - year-div-rate-report # 年度配息率比较
+    real-daily:
+      file-name: svROBO6_portfolios
+      include-report:
+        - daily-hold-report
+        - daily-signal-report
+      email:
+        receives:
+          - wenwen.tang@thizgroup.com
+        copies: ${DAILY_EMAIL_COPIES}
+        subject:
+          default: "ROBO6_TAIBEI-实盘版-每日投組推薦_{today}"
+          rebalance: "ROBO6_TAIBEI-实盘版-每日投組推薦_{today}_今日有調倉信號！！！"
+        content:
+          default: "Dear All: 附件是今天生成的推薦組合，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
+          rebalance: "Dear All: 附件是今天生成的推薦組合以及調倉信號，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
+    daily-monitor:
+      file-name: svROBO6_monitor
+      include-report:
+        - name: relative-range-report # 相对区间收益报告
+          min-date: ~
+        - name: contribution-report # 贡献率报告
+          min-date: {days: 30}
+        - name: high-weight-report # 高风险资产占比
+          min-date: {days: 30}
+        - name: asset-pool-report # 基金池
+          min-date: {days: 30}
+        - name: combo-report # 持仓报告
+          min-date: {days: 40}
+        - name: mpt-report
+          min-date: {days: 30}
+        - name: signal-report
+          min-date: ~
+        - name: crisis-one-report
+          min-date: {days: 30}
+        - name: crisis-two-report
+          min-date: {days: 30}
+        - name: market-right-report
+          min-date: {days: 30}
+        - name: drift-buy-report
+          min-date: {days: 30}
+      email:
+        receives:
+          - wenwen.tang@thizgroup.com
+        copies: ${MONITOR_EMAIL_COPIES}
+        subject: "SVROBO6-实盘版-每日监测_{today}"
+        content: "Dear All: 附件是今天生成的监测数据，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
+robo-executor: # 执行器相关
+  use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
+  sync-data: ${SYNC_DATA:off} # 是否开启同步资料数据
+  backtest: # 回测执行器相关
+    start-date: 2023-01-02 # 回测起始日期
+    end-date: 2023-10-31 # 回测截止日期
+    sealing-period: 10 #调仓封闭期
+    start-step: ${BACKTEST_START_STEP:1} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    clean-up: off
+  real: # 实盘执行器
+    export: ${EXPORT_ENABLE:off} # 是否开启报告
+    start-date: 2023-05-08 # 实盘开始时间
+    include-date: []
+
+
+
+

portfolios/builder.py

@@ -4,7 +4,8 @@ import logging
 from py_jftech import component, autowired, format_date
 from pymysql import IntegrityError, constants
 
-from api import PortfoliosBuilder, PortfoliosRisk, AssetPool, Navs, PortfoliosType, Datum, SolveType, SolverFactory
+from api import PortfoliosBuilder, PortfoliosRisk, AssetPool, Navs, PortfoliosType, Datum, SolveType, SolverFactory, \
+    PortfoliosChecker
 from portfolios.dao import robo_mpt_portfolios as rmp
 
 logger = logging.getLogger(__name__)
@@ -14,11 +15,13 @@ logger = logging.getLogger(__name__)
 class MptPortfoliosBuilder(PortfoliosBuilder):
 
     @autowired
-    def __init__(self, assets: AssetPool = None, navs: Navs = None, datum: Datum = None, factory: SolverFactory = None):
+    def __init__(self, assets: AssetPool = None, navs: Navs = None, datum: Datum = None, factory: SolverFactory = None,
+                 checker: PortfoliosChecker = None):
         self._assets = assets
         self._navs = navs
         self._datum = datum
         self._factory = factory
+        self._checker = checker
 
     def get_portfolios(self, day, risk: PortfoliosRisk, type: PortfoliosType = PortfoliosType.NORMAL):
         try:
@@ -26,6 +29,7 @@ class MptPortfoliosBuilder(PortfoliosBuilder):
             if not portfolio:
                 result = self.build_portfolio(day, type)
                 for build_risk, datas in result.items():
+                    datas['portfolio'] = self._checker.check(day, json.loads(datas['portfolio']))
                     try:
                         rmp.insert({
                             **datas,
@@ -44,7 +48,7 @@ class MptPortfoliosBuilder(PortfoliosBuilder):
             return None
         except Exception as e:
             logger.exception(
-                f"build protfolio of type[{type.name}] and risk[{risk.name}] with date[{format_date(day)}] failure.", e)
+                f"build portfolio of type[{type.name}] and risk[{risk.name}] with date[{format_date(day)}] failure.", e)
             raise e
 
     def build_portfolio(self, day, type: PortfoliosType):
@@ -119,6 +123,7 @@ class MptARCPortfoliosBuilder(MptPortfoliosBuilder):
             if not portfolio:
                 result, detail = self.build_portfolio(day, type)
                 for build_risk, datas in result.items():
+                    datas['portfolio'] = self._checker.check(day, json.loads(datas['portfolio']))
                     try:
                         rmp.insert({
                             **datas,

portfolios/checker.py

+import logging
+
+from py_jftech import autowired, component, get_config
+
+from api import AssetOptimize, PortfoliosChecker, Datum, Navs, DatumType
+
+logger = logging.getLogger(__name__)
+
+
+@component(bean_name='checker')
+class DefaultPortfoliosChecker(PortfoliosChecker):
+
+    @autowired
+    def __init__(self, asset: AssetOptimize = None, navs: Navs = None, datum: Datum = None):
+        self._asset = asset
+        self._navs = navs
+        self._datum = datum
+        self._config = get_config(__name__)
+
+    def check(self, day=None, portfolios: dict = None):
+        if not self._config.get('switch'):
+            return portfolios
+        funds = self._datum.get_datums(type=DatumType.FUND)
+        company = {f"{fund['id']}": fund['companyType'] for fund in funds}
+        customType = {f"{fund['id']}": fund['customType'] for fund in funds}
+        companies = set(company[key] for key in portfolios.keys())
+        # 同时出现全部是ft或美盛基金的情况
+        if len(companies) == 1:
+            # step1:	检查原始投组的customType。检查顺序用列表呈现，依序进行
+            priority = self._config.get('custom-type-priority')
+            for p in priority:
+                keys = [key for key in portfolios.keys() if customType[key] == p]
+                # 若存在匹配值则执行后跳出循环
+                if len(keys) > 0:
+                    ids = [fund['id'] for fund in funds if fund['companyType'] != list(companies)[0]]
+                    best = self.find_highest_score(ids, day)
+                    # 若刚好有一个匹配，直接替换
+                    if len(keys) == 1:
+                        portfolios[best] = portfolios[keys[0]]
+                        # 删除原始键
+                        del portfolios[keys[0]]
+                    else:
+                        # 算分，把分低的替换掉
+                        scores = self.do_score(keys, day)
+                        weight_scores = {key: scores[key]*portfolios[key] for key in keys}
+                        lowest = min(scores, key=lambda k: weight_scores[k])
+                        portfolios[best] = portfolios[lowest]
+                        # 删除原始键
+                        del portfolios[lowest]
+                    break
+        return portfolios
+
+    def do_score(self, ids, day):
+        optimize = self._asset.find_optimize(fund_ids=ids, day=day)
+        scores = optimize[1].to_dict()
+        id_score = {}
+        for k, v in scores.items():
+            id_score[f'{ids[k]}'] = v
+        return id_score
+
+    def find_highest_score(self, ids, day):
+        optimize = self._asset.find_optimize(fund_ids=ids, day=day)
+        return optimize[0][0]