可以指定任意天数的预测

ai/EstimateMarketTrendV20.py

@@ -2,7 +2,7 @@ import json
 from typing import List
 
 import numpy as np
-from py_jftech import autowired, parse_date, prev_workday, format_date
+from py_jftech import autowired, parse_date, prev_workday, format_date, get_config
 
 from ai.config import LABEL_RANGE, LABEL_TAG
 from ai.dao import robo_predict
@@ -16,10 +16,10 @@ from api import DataSync
 # 截止日期
 max_date = None
 
-toForecast = False  # False means test, True means forecast
-syncData = False  # 开启会同步数据库指数及基金数据
-uploadData = False  # 开启会上传预测结果
-doReport = True  # 开启会生成Excel报告
+toForecast = get_config('robo-executor.to-forecast')
+doReport = get_config('robo-executor.export')
+numForecastDays = get_config('robo-executor.num-forecast-days')  # business days, 21 business days means one month
+
 # JIFU_SPX_OPEPS_CURRQ_TTM
 # JIFU_SPX_OPEPS_YMAE1
 # JIFU_SPX_OPEPS_YMAE2
@@ -39,9 +39,11 @@ fund = [156]
 
 @autowired
 def sync(syncs: List[DataSync] = None):
-    for s in syncs:
-        # if isinstance(s, (IndexSync, EcoSync)):
-        s.do_sync()
+    sync_data = get_config('robo-executor.sync-data')
+    if sync_data:
+        for s in syncs:
+            # if isinstance(s, (IndexSync, EcoSync)):
+            s.do_sync()
 
 
 def report_prediction(label, predict_item, indexDict: dict):
@@ -52,29 +54,29 @@ def report_prediction(label, predict_item, indexDict: dict):
     # 上传预测结果
     key = [k for k, v in indexDict.items() if v == predict_item][0]
     index_info = get_base_info(key)[0]
+    uploadData = get_config('robo-executor.upload-data')
     if uploadData:
         if len(LABEL_RANGE) > 2:
-            data = {"rbd_id": key, "date": forecastDay, "predict": prediction}
+            data = {"rbd_id": key, "date": forecastDay, "predict": prediction, "remark": {"period": numForecastDays}}
             robo_predict.insert(data)
         else:
             from ai.noticer import upload_predict
             upload_predict(index_info['ticker'], forecastDay, predictionStr)
-    send(content)
+    # send(content)
     return prediction
 
 
 def judgement(id, type, predict):
     from datetime import datetime
-    predict_term = 21
     start = parse_date(max_date) if max_date else prev_workday(datetime.today())
     navs = []
     if type == 'INDEX':
-        navs = get_index_list(index_ids=id, min_date=start, limit=predict_term + 1)
+        navs = get_index_list(index_ids=id, min_date=start, limit=numForecastDays + 1)
         navs = [nav['rid_close'] for nav in navs]
     elif type == 'FUND':
-        navs = get_fund_list(fund_ids=id, min_date=start, limit=predict_term + 1)
+        navs = get_fund_list(fund_ids=id, min_date=start, limit=numForecastDays + 1)
         navs = [nav['rfn_nav_cal'] for nav in navs]
-    if len(navs) == predict_term + 1:
+    if len(navs) == numForecastDays + 1:
         upper = True if navs[-1] >= navs[0] else False
         result = {}
         for k, v, in predict.items():
@@ -95,13 +97,11 @@ def judgement(id, type, predict):
 
 ########################################
 if __name__ == '__main__':
-    if syncData:
-        sync()
+    sync()
     # define some parameters
     win1W = 5  # 1 week
     win1M = 21  # 1 Month
     win1Q = 63  # 1 Quarter
-    numForecastDays = 21  # business days, 21 business days means one month
     theThreshold = 0.0
     ids = set(PREDICT_LIST) | set(eco) | set(index) | set(fund)
     infos = get_base_info(ids)
@@ -155,7 +155,7 @@ if __name__ == '__main__':
             else:
                 report_prediction(ensemble_model.predict(scaledX_forecast), indexDict[pid], indexDict)
     if doReport:
-        if len(LABEL_RANGE) > 2:
+        if len(LABEL_RANGE) > 2 and toForecast:
             from ai.reporter import do_reporter2
 
             do_reporter2()

ai/model_trainer.py

+import json
 from abc import ABC
 from datetime import datetime
 
 from lightgbm import LGBMClassifier
+from py_jftech import get_config
 from sklearn import svm
 from sklearn.ensemble import AdaBoostClassifier
 from sklearn.ensemble import RandomForestClassifier, VotingClassifier
@@ -41,9 +43,11 @@ class ModelTrainer(ABC):
         if doReport:
             if strMethod == "Ensemble Model":
                 datas = []
+                numForecastDays = get_config('robo-executor.num-forecast-days')
                 for predict, date in zip(y_pred, date_index):
                     datas.append(
-                        {'predict': predict, 'date': date, 'rbd_id': self._pid, 'create_time': datetime.now()})
+                        {'predict': predict, 'date': date, 'rbd_id': self._pid,
+                         "remark": json.dumps({"period": numForecastDays}), 'create_time': datetime.now()})
                 reporter.do_reporter2(records=datas, excel_name='Backtest_Report_chu.xlsx')
 
         # cm_display = ConfusionMatrixDisplay(confusion_matrix=result0, display_labels=labels)

ai/reporter.py

@@ -42,18 +42,20 @@ def do_reporter2(records=None, excel_name=None):
     if not records:
         records = robo_predict.get_list()
     for item in records:
+        predict_term = json.loads(item['remark'])['period']
         result, rtn, real = is_right(item['rbd_id'], info.get(item['rbd_id'])['type'], item['date'],
-                                     item['predict'])
+                                     item['predict'], predict_term)
         labels = list(LABEL_RANGE.keys())[::-1]
         data = {
             'Forcast On Date': item['date'],
             'Ticker': info.get(item['rbd_id'])['ticker'].replace(' Index', '').replace(' Equity', ''),
-            'In 21 business days': LABEL_TAG.get(item['predict']),
+            'Forecast Business Days After': predict_term,
+            'Forecast Label': LABEL_TAG.get(item['predict']),
             'real outcome label': LABEL_TAG.get(real),
             'random variable label': LABEL_TAG.get(random.randint(labels[0], labels[-1])),
             'Ticker Name': json.loads(info.get(item['rbd_id'])['datas'])['chineseName'],
             'Run On Time': item['create_time'].replace(minute=0, second=0),
-            'return in 21 business days': rtn,
+            'return on Forecast Label': rtn,
             'result': result
         }
         datas.append(data)
@@ -66,7 +68,9 @@ def do_reporter2(records=None, excel_name=None):
     pf['RMSE'] = RMSE
     pf['RefRMSE'] = RefRMSE
     pf['RevisedReferenceRMSE'] = RevisedReferenceRMSE
-    pf.to_excel(excel_name if excel_name else "Forcast_Report_chu.xlsx", index=False)
+    excel_name = excel_name if excel_name else "Forcast_Report_chu.xlsx"
+    with pd.ExcelWriter(excel_name, mode='a', if_sheet_exists='replace') as writer:
+        pf.to_excel(writer, sheet_name=f'{predict_term} business days', index=False)
 
 
 def map_to_label(ret):
@@ -82,9 +86,11 @@ def get_RMSE(datas):
     @return:
     """
     datas = [data for data in datas if data.get('real outcome label') is not None]
+    if not datas:
+        return 0
     tags = {v: k for k, v in LABEL_TAG.items()}
     return (sum(
-        [(tags.get(data['real outcome label']) - tags.get(data['In 21 business days'])) ** 2 for data in datas]) / len(
+        [(tags.get(data['real outcome label']) - tags.get(data['Forecast Label'])) ** 2 for data in datas]) / len(
         datas)) ** 0.5
 
 
@@ -95,6 +101,8 @@ def get_RefRMSE(datas):
     @return:
     """
     datas = [data for data in datas if data.get('real outcome label') is not None]
+    if not datas:
+        return 0
     tags = {v: k for k, v in LABEL_TAG.items()}
     return (sum(
         [(tags.get(data['real outcome label']) - tags.get(data['random variable label'])) ** 2 for data in
@@ -131,8 +139,7 @@ def get_RevisedReferenceRMSE(datas):
     return revised_ref_rmse
 
 
-def is_right(id, type, start, predict):
-    predict_term = 21
+def is_right(id, type, start, predict, predict_term):
     navs = []
     if type == 'INDEX':
         navs = get_index_list(index_ids=id, min_date=start, limit=predict_term + 1)

config-ai.yml

+py-jftech:
+  logger:
+    version: 1
+    formatters:
+      brief:
+        format: "%(asctime)s - %(levelname)s - %(message)s"
+      simple:
+        format: "%(asctime)s - %(filename)s - %(levelname)s - %(message)s"
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        level: DEBUG
+        stream: ext://sys.stdout
+      file:
+        class: logging.handlers.TimedRotatingFileHandler
+        level: INFO
+        formatter: brief
+        filename: ${LOG_FILE:logs/info.log}
+        interval: 1
+        backupCount: 30
+        encoding: utf8
+        when: D
+#    loggers:
+#      basic.sync:
+#        level: DEBUG
+#        handlers: [console]
+#        propagate: no
+    root:
+      level: ${LOG_LEVEL:INFO}
+      handlers: ${LOG_HANDLERS:[ console ]}
+  database:
+    host: ${MYSQL_HOST:106.14.56.221}
+    port: ${MYSQL_PORT:3306}
+    user: ${MYSQL_USER:ai_user}
+    password: ${MYSQL_PWD:ai.predict*2025}
+    dbname: ${MYSQL_DBNAME:ai_predict}
+  injectable:
+    names:
+      backtest: robo_executor.BacktestExecutor
+      datum: basic.datum.DefaultDatum
+      hold-report: portfolios.holder.DivHoldReportor
+      mpt: portfolios.builder.PoemARCPortfoliosBuilder
+      dividend-holder: portfolios.holder.InvTrustPortfoliosHolder
+      navs-sync: basic.sync.FundNavSync
+  email:
+    server: smtphz.qiye.163.com
+    user: jft-ra@thizgroup.com
+    password: 5dbb#30ec6d3
+  mulit-process:
+    max-workers: ${MAX_PROCESS:4}
+basic: # 基础信息模块
+  sync:
+    start-date: 2018-08-26 # 同步数据开始日期
+  datum: # 资料模块
+    change:
+      date: ${DATUM_CHANGE_DATE}
+      file: ${DATUM_CHANGE_FILE}
+    excludes: # 排除的资料彭博ticker
+#      backtest:
+#        - 'TEMUSGI LX Equity'
+      real:
+        - 'FGFSACU LX Equity'
+        - 'TEMUSGI LX Equity'
+  navs: # 净值模块
+    exrate: # 汇率，如果不开启，整个这块注释掉
+      - from: EUR # 需要转换的货币类型
+        ticker: EURUSD BGN Curncy # 汇率值的彭博ticker
+robo-executor: # 执行器相关
+  to-forecast: off  # False means test, True means forecast
+  sync-data: ${SYNC_DATA:off} # 开启会同步数据库指数及基金数据
+  upload-data: off  # 开启会上传预测结果
+  export: ${EXPORT_ENABLE:on} # 是否开启报告
+  num-forecast-days: 21 # business days, 21 business days means one month
+
+  use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
+  backtest: # 回测执行器相关
+    start-date: 2018-11-26 # 回测起始日期
+    end-date: 2019-01-13 # 回测截止日期
+    sealing-period: 10 #调仓封闭期
+    start-step: ${BACKTEST_START_STEP:1} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
+    clean-up: off
+  real: # 实盘执行器
+    export: ${EXPORT_ENABLE:off} # 是否开启报告
+    start-date: 2023-05-08 # 实盘开始时间
+    include-date: []
+
+
+
+

config-svrobo6.yml

-py-jftech:
-  logger:
-    version: 1
-    formatters:
-      brief:
-        format: "%(asctime)s - %(levelname)s - %(message)s"
-      simple:
-        format: "%(asctime)s - %(filename)s - %(levelname)s - %(message)s"
-    handlers:
-      console:
-        class: logging.StreamHandler
-        formatter: simple
-        level: DEBUG
-        stream: ext://sys.stdout
-      file:
-        class: logging.handlers.TimedRotatingFileHandler
-        level: INFO
-        formatter: brief
-        filename: ${LOG_FILE:logs/info.log}
-        interval: 1
-        backupCount: 30
-        encoding: utf8
-        when: D
-#    loggers:
-#      basic.sync:
-#        level: DEBUG
-#        handlers: [console]
-#        propagate: no
-    root:
-      level: ${LOG_LEVEL:INFO}
-      handlers: ${LOG_HANDLERS:[ console ]}
-  database:
-    host: ${MYSQL_HOST:106.14.56.221}
-    port: ${MYSQL_PORT:3306}
-    user: ${MYSQL_USER:ai_user}
-    password: ${MYSQL_PWD:ai.predict*2025}
-    dbname: ${MYSQL_DBNAME:ai_predict}
-  injectable:
-    names:
-      backtest: robo_executor.BacktestExecutor
-      datum: basic.datum.DefaultDatum
-      hold-report: portfolios.holder.DivHoldReportor
-      mpt: portfolios.builder.PoemARCPortfoliosBuilder
-      dividend-holder: portfolios.holder.InvTrustPortfoliosHolder
-      navs-sync: basic.sync.FundNavSync
-  email:
-    server: smtphz.qiye.163.com
-    user: jft-ra@thizgroup.com
-    password: 5dbb#30ec6d3
-  mulit-process:
-    max-workers: ${MAX_PROCESS:4}
-basic: # 基础信息模块
-  sync:
-    start-date: 2018-08-26 # 同步数据开始日期
-  datum: # 资料模块
-    change:
-      date: ${DATUM_CHANGE_DATE}
-      file: ${DATUM_CHANGE_FILE}
-    excludes: # 排除的资料彭博ticker
-#      backtest:
-#        - 'TEMUSGI LX Equity'
-      real:
-        - 'FGFSACU LX Equity'
-        - 'TEMUSGI LX Equity'
-  navs: # 净值模块
-    exrate: # 汇率，如果不开启，整个这块注释掉
-      - from: EUR # 需要转换的货币类型
-        ticker: EURUSD BGN Curncy # 汇率值的彭博ticker
-asset-pool: # 资产池模块
-  asset-optimize: # 资产优选模块
-    sortino-weight: # sortino计算需要的权重，下面每一条为一次计算，e.g. months: 3, weight: 0.5 表示 3个月数据使用权重0.5来计算分值
-      - months: 3
-        weight: 0.5
-      - months: 6
-        weight: 0.3
-      - years: 1
-        weight: 0.2
-    asset-include: {'customType':[1,2,3,4]}
-    optimize-count: 3 #基金优选个数
-portfolios: # 投组模块
-  holder: # 持仓投组相关
-    init-nav: 100 # 初始金额
-    min-interval-days: 10 # 两次实际调仓最小间隔期，单位交易日
-    dividend-rate: 0.09 #设定年化配息率
-    dividend-date: 15 #配息日，每月15号
-    dividend-adjust-day: [1,4,7,10]  #每年的首个季度调整配息
-    warehouse-frequency: 1  #每隔1个月调一次仓
-    warehouse-transfer-date: 1  #调仓日
-    redeem-list: [ 'TEUSAAU LX Equity', 'LIGTRAA ID Equity', 'TEMFHAC LX Equity', 'LUSHUAA ID Equity' ] #从持仓中的低风险资产“直接”按序赎回
-  solver: # 解算器相关
-    model: prr # 结算模型 ARC ,PRR,  ~ 标准解算器
-    arc: on  #是否开启ARC
-    brr: 0.02 #误差补偿值
-    trr: 3
-    tol: 1E-10 # 误差满足条件
-    navs: # 净值要求
-      range: # 需要净值数据的区间, days: 90 表示90自然日，months: 3 表示3个自然月
-        days: 90
-      max-nan: # 最大缺失净值条件
-        asset: 8 # 单一资产最多缺少多少交易日数据，则踢出资产池
-        day: 0.5 # 单一交易日最多缺少百分之多少净值，则删除该交易日
-      risk: [ ]  # 资产风险等级要求，可分开写也可以合并写，e.g. risk：[ 2, 3 ] 则表示 所有投组资产风险等级都是 2 或 3
-    LARC: [ 0.30, 0.00, 0.00 ]  #低阈值
-    UARC: [ 0.70, 0.70, 0.70 ] #高阈值
-    matrix-rtn-days: 20 # 计算回报率矩阵时，回报率滚动天数
-    asset-count: [5,5] # 投组资产个数。e.g. count 或 [min, max] 分别表示 最大最小都为count 或 最小为min 最大为max，另外这里也可以类似上面给不同风险等级分别配置
-    mpt: # mpt计算相关
-      cvar-beta: 0.2 # 计算Kbeta 需要用到
-      quantile: 0.9 # 分位点，也可以给不同风险等级分别配置
-      low-weight: 0.05 # 最低权重
-      high-weight: [ 0.35 ] # 最高权重比例，可给一个值，也可以给多个值，当多个值时，第一个表示只有一个资产时权重，第二个表示只有两个资产时权重，以此类推，最后一个表示其他资产个数时的权重
-    poem: # poem相关
-      cvar-scale-factor: 0.1 # 计算时用到的系数
-  checker: #投组检测模块
-    switch: on  #是否开启检查
-    custom-type-priority: [ 3,2,1,4 ]  # 检测优先级
-reports: # 报告模块相关
-  navs:
-    type: FUND
-    tickers:
-      - TEMTECI LX Equity
-      - TEPLX US Equity
-      - FRDPX US Equity
-      - FKRCX US Equity
-      - FTNRACU LX Equity
-  benchmark: # benchmark报告
-    ft:
-      init-amount: 100 # 初始金额
-      stock-rate: # stock型基金比例
-        RR3: 0.3
-        RR4: 0.5
-        RR5: 0.7
-  fixed-range: # 固定区间收益率
-    range-dates: # 固定起始截止日期
-      - start: 2008-01-01
-        end: 2008-10-27
-      - start: 2011-05-02
-        end: 2011-10-04
-      - start: 2013-05-08
-        end: 2013-06-24
-      - start: 2014-09-03
-        end: 2014-12-16
-      - start: 2015-04-28
-        end: 2016-01-21
-      - start: 2018-01-26
-        end: 2018-10-29
-      - start: 2020-01-20
-        end: 2020-03-23
-  relative-range: # 相对区间收益率
-    range-dates: # 相对时间周期
-      - days: 1
-        name: '一天'
-      - weeks: 1
-        name: '一周'
-      - months: 1
-        name: '一月'
-      - months: 3
-        name: '三月'
-      - months: 6
-        name: '六月'
-      - years: 1
-        name: '一年'
-      - years: 2
-        name: '两年'
-      - years: 3
-        name: '三年'
-      - years: 5
-        name: '五年'
-      - years: 10
-        name: '十年'
-      - dates: ~
-        name: '成立以来'
-  exports:
-    backtest: # 回测导出曹策略
-      save-path: ${EXPORT_PATH:excels} # 导出报告文件存放路径，如果以./或者../开头，则会以执行python文件为根目录，如果以/开头，则为系统绝对路径，否则，以项目目录为根目录
-      file-name: ${EXPORT_FILENAME:real} # 导出报告的文件名
-      save-config: ${EXPORT_CONFIG:off} # 是否保存配置文件
-      include-report: # 需要导出的报告类型列表，下面的顺序，也代表了excel中sheet的顺序
-  #      - funds-report # 基金资料
-  #      - navs-report # 净值报告
-        - hold-report # 持仓报告
-        - signal-report # 信号报告
-        - benckmark-report # benckmark报告
-        - combo-report # 持仓对比
-        - indicators-report # 各种特殊指标报告
-        - fixed-range-report # 固定区间收益报告
-        - relative-range-report # 相对区间收益报告
-        - year-range-report # 单年区间业绩报告
-        - month-div-rate-report # 月度配息率比较
-        - year-div-rate-report # 年度配息率比较
-    real-daily:
-      file-name: svROBO6_portfolios
-      include-report:
-        - daily-hold-report
-        - daily-signal-report
-      email:
-        receives:
-          - wenwen.tang@thizgroup.com
-        copies: ${DAILY_EMAIL_COPIES}
-        subject:
-          default: "ROBO6_TAIBEI-实盘版-每日投組推薦_{today}"
-          rebalance: "ROBO6_TAIBEI-实盘版-每日投組推薦_{today}_今日有調倉信號！！！"
-        content:
-          default: "Dear All: 附件是今天生成的推薦組合，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
-          rebalance: "Dear All: 附件是今天生成的推薦組合以及調倉信號，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
-    daily-monitor:
-      file-name: svROBO6_monitor
-      include-report:
-        - name: relative-range-report # 相对区间收益报告
-          min-date: ~
-        - name: contribution-report # 贡献率报告
-          min-date: {days: 30}
-        - name: high-weight-report # 高风险资产占比
-          min-date: {days: 30}
-        - name: asset-pool-report # 基金池
-          min-date: {days: 30}
-        - name: combo-report # 持仓报告
-          min-date: {days: 40}
-        - name: mpt-report
-          min-date: {days: 30}
-        - name: signal-report
-          min-date: ~
-        - name: crisis-one-report
-          min-date: {days: 30}
-        - name: crisis-two-report
-          min-date: {days: 30}
-        - name: market-right-report
-          min-date: {days: 30}
-        - name: drift-buy-report
-          min-date: {days: 30}
-      email:
-        receives:
-          - wenwen.tang@thizgroup.com
-        copies: ${MONITOR_EMAIL_COPIES}
-        subject: "SVROBO6-实盘版-每日监测_{today}"
-        content: "Dear All: 附件是今天生成的监测数据，請驗收，謝謝！ 注>：該郵件為自動發送，如有問題請聯繫矽谷團隊 telan_qian@chifufund.com"
-robo-executor: # 执行器相关
-  use: ${ROBO_EXECUTOR:backtest} # 执行哪个执行器，优先取系统环境变量ROBO_EXECUTOR的值，默认backtest
-  sync-data: ${SYNC_DATA:off} # 是否开启同步资料数据
-  backtest: # 回测执行器相关
-    start-date: 2018-11-26 # 回测起始日期
-    end-date: 2019-01-13 # 回测截止日期
-    sealing-period: 10 #调仓封闭期
-    start-step: ${BACKTEST_START_STEP:1} # 回测从哪一步开始执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
-    end-step: ${BACKTEST_END_STEP:3} # 回测从哪一步执行完成后结束执行 1:计算资产池；2:计算最优投组：3:计算再平衡信号以及持仓投组
-    clean-up: off
-  real: # 实盘执行器
-    export: ${EXPORT_ENABLE:off} # 是否开启报告
-    start-date: 2023-05-08 # 实盘开始时间
-    include-date: []
-
-
-
-