commit d3e43efd637b334e54277e1e0f295c7bb1e9b501 Author: liaomaowen Date: Fri Sep 30 15:59:12 2022 +0800 enjoy coding diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0b8916d --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +venv +pandoc-2.19.2 +docs +__pycache__ diff --git a/convert_to_feishu_online_docs.py b/convert_to_feishu_online_docs.py new file mode 100644 index 0000000..40fa215 --- /dev/null +++ b/convert_to_feishu_online_docs.py @@ -0,0 +1,17 @@ +import json +from feishu import convert + +with open('feishu_file_tokens.json', 'r', encoding='utf-8') as f: + file_tokens = f.read() + + +file_tokens = json.loads(file_tokens) +for t in file_tokens: + 'https://tuc3es5ukv.feishu.cn/docx/RWcBdTuX4oQmHtxdsaUcfX6fnqh' + if '/docx/' in t: + continue + if '/file/' in t: + file_token = t.split('/')[-1] + # print(file_token) + convert(file_token) + diff --git a/export.py b/export.py new file mode 100644 index 0000000..0823bdf --- /dev/null +++ b/export.py @@ -0,0 +1,85 @@ +import os +import re +import json +from logging import root +import requests + +from urllib.parse import urljoin + +root_url = 'https://www.yuque.com/api/v2/' + +token = '8g04jZ0TaAmA7FQ95MhoNHkRsTntNWYcQ8hOor9f' + +headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', + + 'X-Auth-Token': token +} + + +def get_user(): + url = urljoin(root_url, 'user') + resp = requests.get(url, headers=headers) + return resp.json() + +def get_repo(user_id, name): + url = urljoin(root_url, f'users/{user_id}/repos') + # print(url) + resp = requests.get(url, headers=headers) + result = resp.json() + for d in result.get('data', []): + if d['name'] == name: + # print(d) + return d['id'] + +def get_docs(repo_id): + url = urljoin(root_url, f'repos/{repo_id}/docs') + resp = requests.get(url, headers=headers) + result = resp.json() + return result + + + +def save_docs_list_to_json(name, namespace): + user = get_user() + # print(user) + user_id = user.get('data', {}).get('id') + repo_id = get_repo(user_id, ) + docs = get_docs(repo_id) + # print(docs) + + with open('docs_list.json', 'w', encoding='utf-8') as f: + f.write(json.dumps(docs, ensure_ascii=False, indent=4)) + + +def save_docs(namespace): + + save_docs_dir = 'docs' + if not os.path.exists(save_docs_dir): + os.makedirs(save_docs_dir) + + + with open('docs_list.json', 'r', encoding='utf-8') as f: + docs_list = f.read() + + docs_list = json.loads(docs_list).get('data', []) + # docs_list = sorted(docs_list, lambda x: x['created_at']) + for d in docs_list: + slug = d['slug'] + title = d['title'] + url = urljoin(root_url, f'repos/{namespace}/docs/{slug}') + resp = requests.get(url, headers=headers, params={'raw': 1}) + result = resp.json() + markdown_content = result.get('data', {}).get('body', '') + # 正则去除语雀导出的标签 + markdown_content = re.sub("","", markdown_content) + title = title.replace('/', ',') + with open(os.path.join(save_docs_dir, f'{title}.md'), 'w', encoding='utf-8') as f: + f.write(markdown_content) + + +if __name__ == '__main__': + name = '网络爬虫技术' + namespace = 'erliang-vtlak/aufz2f' + save_docs_list_to_json(name=name) + save_docs(namespace) diff --git a/feishu.py b/feishu.py new file mode 100644 index 0000000..5fb1995 --- /dev/null +++ b/feishu.py @@ -0,0 +1,109 @@ +import traceback +import requests + +def convert(file_token): + cookies = { + 'passport_web_did': '7130549151531827201', + 'QXV0aHpDb250ZXh0': 'e6247b47f5fe4f4797bdd1418288e3ce', + 'locale': 'zh-CN', + 'trust_browser_id': 'ec126e85-f8df-4a8c-8f21-cffb11665839', + '__tea__ug__uid': '7130549122386855428', + 'is_anonymous_session': '', + 'lang': 'zh', + 'MONITOR_WEB_ID': '8c557655-1816-4759-91ad-17e471b73905', + '_csrf_token': '9c1e5ab08685598e74d52a4dc61036642619b168-1662979237', + 'help_center_session': 'c126c9d2-19e9-4442-b265-270d66a7a42e', + '_uuid_hera_ab_path_1': '7143827400751710236', + 'vt': '1', + 'et': '42187fdf6599d0d4105764f071fd5711', + 'ot': '06ec70c5edd6a6c6ef054a600ea23021', + 'landing_url': 'https://www.feishu.cn/product/docs', + 'Hm_lvt_e78c0cb1b97ef970304b53d2097845fd': '1663301934,1664505254', + '_gid': 'GA1.2.127991392.1664505254', + 'session': 'XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA', + 'session_list': 'XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA_XN0YXJ0-15fgec87-c20d-4ded-808c-c18680fb20c2-WVuZA_XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA', + 'js_version': '1', + 'Hm_lpvt_e78c0cb1b97ef970304b53d2097845fd': '1664506282', + 'sl_session': 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NjQ1NTAwNjcsInVuaXQiOiJldV9uYyIsInJhdyI6eyJtZXRhIjoiQVdCY1h3eHkwa0FFWUZ4ZkRHY0J3QVFBQUFBQUFBQUFBR0wwekt5T2hJQUJBQUFBQUFBQUFBQUNLZ0VBUVVGQlFVRkJRVUZCUVVKcVJVMW5lbkJCUVVGSVFUMDkiLCJpZGMiOlsxLDJdLCJzdW0iOiJiYmQwODVlOGQ1OGMwZDkyZmM2NzdkZGE3NWFiZTk4Yjg3YWUwZWU4YTUwZjRmZmM4MmQxZmMxMTM1N2Y0YzQ5IiwibG9jIjoiemhfY24iLCJhcGMiOiJSZWxlYXNlIiwiaWF0IjoxNjY0NTA2ODY3fX0.NzVIL4Rd2LSZRggZHP4kcC4mcJ7-EAfi5KYgr__lIzzM9h38chC1THEeyh0DBjKAsybCB4JWKoFD5nuXMhABEQ', + '_ga': 'GA1.2.1604574774.1660210350', + '_ga_VPYRHN104D': 'GS1.1.1664505254.3.1.1664507428.0.0.0', + 'swp_csrf_token': 'c3f2e4b0-d6bb-4adc-b4d2-205e9705189d', + 't_beda37': '3e03e2c3de3e4313706401de8810ef4c7614e174d84d3c22488d8670695d5288', + } + + + headers = { + 'authority': 'tuc3es5ukv.feishu.cn', + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'en,zh-CN;q=0.9,zh;q=0.8', + 'cache-control': 'no-cache', + 'context': 'request_id=Hk5frCxBNMgx-f4f5e1c3f7e5721521ca196a63ead3d27e058b84;os=windows;app_version=1.0.7.8782;os_version=10;platform=web', + 'doc-biz': 'Lark', + 'doc-os': 'windows', + 'doc-platform': 'web', + 'f-version': 'docs-9-28-1664352681274', + 'origin': 'https://tuc3es5ukv.feishu.cn', + 'pragma': 'no-cache', + 'referer': f'https://tuc3es5ukv.feishu.cn/file/{file_token}', + 'request-id': 'Hk5frCxBNMgx-f4f5e1c3f7e5721521ca196a63ead3d27e058b84', + 'sec-ch-ua': '"Google Chrome";v="105", "Not)A;Brand";v="8", "Chromium";v="105"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36', + 'x-csrftoken': '9c1e5ab08685598e74d52a4dc61036642619b168-1662979237', + 'x-request-id': 'Hk5frCxBNMgx-f4f5e1c3f7e5721521ca196a63ead3d27e058b84', + 'x-tt-logid': '02166450822736900000000000000000000ffffaa9463ac5a5ec6', + 'x-tt-trace-id': 'Hk5frCxBNMgx-f4f5e1c3f7e5721521ca196a63ead3d27e058b84', + } + + json_data = { + 'file_token': file_token, + 'type': 'docx', + 'file_extension': 'md', + 'point': { + 'mount_type': 1, + 'mount_key': 'fldcnYylqxivPSlPzF5SCHKpdje', + }, + 'event_source': '3', + 'passback': '{"time_zone":"Etc/GMT-8"}', + } + try: + response = requests.post('https://tuc3es5ukv.feishu.cn/space/api/import/create/', cookies=cookies, headers=headers, json=json_data) + print(response.json()) + except: + traceback.print_exc() + + +import requests + +cookies = { + 'passport_web_did': '7130549151531827201', + 'QXV0aHpDb250ZXh0': 'e6247b47f5fe4f4797bdd1418288e3ce', + 'locale': 'zh-CN', + 'trust_browser_id': 'ec126e85-f8df-4a8c-8f21-cffb11665839', + '__tea__ug__uid': '7130549122386855428', + 'is_anonymous_session': '', + 'lang': 'zh', + 'MONITOR_WEB_ID': '8c557655-1816-4759-91ad-17e471b73905', + '_csrf_token': '9c1e5ab08685598e74d52a4dc61036642619b168-1662979237', + 'help_center_session': 'c126c9d2-19e9-4442-b265-270d66a7a42e', + '_uuid_hera_ab_path_1': '7143827400751710236', + 'vt': '1', + 'et': '42187fdf6599d0d4105764f071fd5711', + 'ot': '06ec70c5edd6a6c6ef054a600ea23021', + 'landing_url': 'https://www.feishu.cn/product/docs', + 'Hm_lvt_e78c0cb1b97ef970304b53d2097845fd': '1663301934,1664505254', + '_gid': 'GA1.2.127991392.1664505254', + 'session': 'XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA', + 'session_list': 'XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA_XN0YXJ0-15fgec87-c20d-4ded-808c-c18680fb20c2-WVuZA_XN0YXJ0-dcdr7041-4255-4d02-8b18-c01128c3f6cf-WVuZA', + 'js_version': '1', + 'Hm_lpvt_e78c0cb1b97ef970304b53d2097845fd': '1664506282', + '_ga': 'GA1.2.1604574774.1660210350', + '_ga_VPYRHN104D': 'GS1.1.1664511776.5.1.1664511777.0.0.0', + 'sl_session': 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2NjQ1NjAwMDksInVuaXQiOiJldV9uYyIsInJhdyI6eyJtZXRhIjoiQVdCY1h3eHkwa0FFWUZ4ZkRHY0J3QVFBQUFBQUFBQUFBR0wwekt5T2hJQUJBQUFBQUFBQUFBQUNLZ0VBUVVGQlFVRkJRVUZCUVVKcVJVMW5lbkJCUVVGSVFUMDkiLCJpZGMiOlsxLDJdLCJzdW0iOiJiYmQwODVlOGQ1OGMwZDkyZmM2NzdkZGE3NWFiZTk4Yjg3YWUwZWU4YTUwZjRmZmM4MmQxZmMxMTM1N2Y0YzQ5IiwibG9jIjoiemhfY24iLCJhcGMiOiJSZWxlYXNlIiwiaWF0IjoxNjY0NTE2ODA5fX0.26KQ2kgVbi3GVz0Pu4KPhIf1sothuZXGa2bTyAv7nuUDdNfcy3I4OLKZN6SpeefRBQAMyRXwciI5KtNG4WpQdg', + 'swp_csrf_token': '8075b7f1-2df5-4ee2-9209-3edc91805517', + 't_beda37': 'd52280435f3eda5b89b170db3023e1fa6a912d760dc72a11c35e67f470a0de62', +} diff --git a/feishu_file_tokens.json b/feishu_file_tokens.json new file mode 100644 index 0000000..ad16bed --- /dev/null +++ b/feishu_file_tokens.json @@ -0,0 +1,3 @@ +[ + "https://tuc3es5ukv.feishu.cn/docx/you file token", +] \ No newline at end of file diff --git a/observe_div.js b/observe_div.js new file mode 100644 index 0000000..3897e8c --- /dev/null +++ b/observe_div.js @@ -0,0 +1,24 @@ +mydocs = document.querySelector("#mainContainer > div.app-main-container.flex.layout-row.explorer-v3 > div > div.sc-llYToB.bepCke > main > div.sc-ewSSRw.fDTLhF > div.sc-eicnZh.bSQUzR > div.sc-fvxABq.eCsope.explorer-file-list-virtualized__container.explorer-file-list-virtualized__container-a > div:nth-child(1) > div") + +DocumentObserverConfig = { + attributes: true, + childList: true, + characterData: true, + subtree: true +}; + +hrefs = [] +DocumentObserver = new MutationObserver(function() { + var items = mydocs.getElementsByTagName('a') + for (let i of items ){ + hrefs.push(i.href) + } +}); + +DocumentObserver.observe(mydocs, DocumentObserverConfig) + +function unique (arr) { + return Array.from(new Set(arr)) +} + +result = unique(hrefs) \ No newline at end of file