宏观数据爬取

ai/spider.py

+import requests
+from lxml import html
+
+
+class Spider:
+    def __init__(self):
+        self.UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0'
+        headers = {
+            'User-Agent': self.UA,
+            'Accept': '*/*',
+            'Connection': 'keep-alive'
+        }
+        url = 'https://en.macromicro.me/collections/34/us-stock-relative/444/us-mm-gspc'
+        response = requests.request("GET", url, headers=headers)
+        # 获取cookie
+        cookies = response.cookies
+        PHPSESSID = cookies['PHPSESSID']
+        tree = html.fromstring(response.content)
+        auth = tree.xpath('//*[@id="panel"]/footer/div[2]/div/p[1]')[0].get('data-stk')
+        self._auth = auth
+        self._PHPSESSID = PHPSESSID
+
+    def fetch(self, code):
+        url = f"https://en.macromicro.me/charts/data/{code}"
+        headers = {"authorization": f"Bearer {self._auth}",
+                   "cookie": f"PHPSESSID={self._PHPSESSID}",
+                   "referer": "https://en.macromicro.me", "Content-Type": "application/json",
+                   "User-Agent": self.UA,
+                   "Accept-Encoding": "br, deflate, gzip, x-gzip",
+                   "Accept": "*/*", "content-length": "0"}
+        response = requests.request("GET", url, headers=headers)
+        return response.json()
+
+
+if __name__ == '__main__':
+    # 实时去取数据
+    # Citigroup US Earnings RevisionIndexes 55746
+    # Macromicro US stock fundementals index 444
+    # The CNN Fear and Greed Index 50108
+
+    # Citigroup US Earnings RevisionIndexes		https://en.macromicro.me/collections/3208/high-frequency-data/55746/us-eu-jp-citi-earnings-revision-index
+    # sp500 trailing twelve months (TTM) average eps
+    # S&P 500 EPS forward concensus estimates of the current and forthcoming three quarters
+    # Macromicro US stock fundementals index	发布日	https://en.macromicro.me/collections/34/us-stock-relative/444/us-mm-gspc
+    # The CNN Fear and Greed Index		https://en.macromicro.me/charts/50108/cnn-fear-and-greed
+    # US CCC Credit Spread (rated CCC yields minus 10Y yield)		https://en.macromicro.me/collections/384/spreads/930/us-credit-spread
+    # The cyclically adjusted price-earnings ratio (CAPE ratio)
+    # The US total stock market capitalization divided by M2 money supply
+    # G20 OECD total leading indicators  https://oecdch.art/api/public/chartConfig/d8b9c2a1f0/G20/-/2008-01/2024-01?preParsedDataOnly&lang=default&version=2
+    print(Spider().fetch(444))