diff --git a/jason78989/.gitignore b/jason78989/.gitignore new file mode 100644 index 00000000..6fa4ec11 --- /dev/null +++ b/jason78989/.gitignore @@ -0,0 +1,9 @@ +**/*.del +**/tmp_* +**/*.m4a +**/*.db +**/*.part +**/*.ytdl +**/cookies.txt +**/url_ignore.txt +**/last_run.txt \ No newline at end of file diff --git a/jason78989/gitUpdate.bat b/jason78989/gitUpdate.bat new file mode 100644 index 00000000..e188928b --- /dev/null +++ b/jason78989/gitUpdate.bat @@ -0,0 +1,7 @@ +git status . + +@pause + +git add . +git commit -m"update jason78989," +start git push \ No newline at end of file diff --git a/jason78989/meta.md b/jason78989/meta.md new file mode 100644 index 00000000..73eb8fe7 --- /dev/null +++ b/jason78989/meta.md @@ -0,0 +1,56 @@ +--- +tags: [docker, scraping, twitter] +--- + +# jason78989 + +jason78989 + +https://filetransfer.io/data-package/w6pFWHek#link + +## references: + + - https://github.com/HoloArchivists/twspace-dl + - https://github.com/HitomaruKonpaku/twspace-crawler + - https://blog.maki0419.com/2022/01/youtube-download-ytdlp-ffmpeg.html + +## twitter link + + - 韭菜兄弟 + - user id: cryptoleek + - https://twitter.com/i/spaces/1kvJpvwXDooKE + + - 枪十七 + - user id: 0xjuu_17 + - https://twitter.com/i/spaces/0xjuu_17/status/1738911874077643115 + - space link -> https://twitter.com/i/spaces/1LyxBnmNkObxN + +> 無問題 🙇🏻‍♂️❤️ +> 好似要 Twitter 嘅 Premium 先可以開 +> 我嗰陣時留個 message 俾你，唔好意思打攪你休息 + +> https://x.com/0xjuu_17/status/1738911874077643115 + +冇問題呀，其實我想試下先再報比你做唔做到 +script 已經做左， +我想睇下攞唔攞到佢條 live 或啫會唔會攞漏左咁解 + +或啫你識開 live 的話你開一開比我夾下都可以 + +如果冇問題先再報比你， +我估我會報大概 HKD500, 你睇睇有冇問題？ + +> 我大概做左個 draft, 佢會 mon 實個 space 同埋錄低個 space. +> 個 program 會係 background 度行， +> 你平時行住的話搵到有 live 就會自己 download 同埋通知你(discord) + +如果想要埋 video 的話，技術上係可行嘅， +但係電腦資源果方面可能要多小小， +如果你係用日常部機去做的話可能成件事會冇咁企理 +(同埋價錢上可能要調整一下) + +> 我諗我會報 HKD500 +> 價錢方面如果你啱的話 +> 開個 discord call 我 remote 你部機去做個 setup +> 時間方面可以就你 (但係唔好夜過 12 點就得喇) +> 應該大約一個鐘頭可以 setup 完 diff --git a/jason78989/package.json b/jason78989/package.json new file mode 100644 index 00000000..23a61869 --- /dev/null +++ b/jason78989/package.json @@ -0,0 +1,12 @@ +{ + "name": "jason78989", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC" +} diff --git a/jason78989/project/.editorconfig b/jason78989/project/.editorconfig new file mode 100644 index 00000000..905b69a5 --- /dev/null +++ b/jason78989/project/.editorconfig @@ -0,0 +1,16 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/jason78989/project/.env.example b/jason78989/project/.env.example new file mode 100644 index 00000000..00e5d22a --- /dev/null +++ b/jason78989/project/.env.example @@ -0,0 +1,5 @@ +TW_USERNAME= +TW_PASSWORD= +TW_EMAIL= +TW_EMAIL_PASSWORD= +DISCORD_WEBOOOK_URL= \ No newline at end of file diff --git a/jason78989/project/.prettierrc b/jason78989/project/.prettierrc new file mode 100644 index 00000000..1df587de --- /dev/null +++ b/jason78989/project/.prettierrc @@ -0,0 +1,27 @@ +{ + "arrowParens": "avoid", + "bracketSpacing": true, + "htmlWhitespaceSensitivity": "strict", + "insertPragma": false, + "jsxBracketSameLine": false, + "jsxSingleQuote": false, + "printWidth": 120, + "proseWrap": "preserve", + "quoteProps": "as-needed", + "requirePragma": false, + "semi": true, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "all", + "useTabs": false, + "overrides": [ + { + "files": ["*.html"], + "options": {} + }, + { + "files": ["*.{j,t}sx", "*.{j,t}s"], + "options": {} + } + ] +} diff --git a/jason78989/project/Dockerfile b/jason78989/project/Dockerfile new file mode 100644 index 00000000..7d8251ef --- /dev/null +++ b/jason78989/project/Dockerfile @@ -0,0 +1,32 @@ +FROM ubuntu:22.04 + +# Set DEBIAN_FRONTEND to noninteractive +ENV DEBIAN_FRONTEND noninteractive + +# RUN sed -i 's/http:\/\/archive\.ubuntu\.com/http:\/\/ftp\.cuhk\.edu\.hk\/pub\/Linux/g' /etc/apt/sources.list && \ +# sed -i 's/http:\/\/security\.ubuntu\.com/http:\/\/ftp\.cuhk\.edu\.hk\/pub\/Linux/g' /etc/apt/sources.list + +USER root + +RUN apt-get update && \ + apt-get install -y zsh sudo curl gnupg2 + +RUN apt-get install -qqy python3 python3-pip + +RUN apt-get install -qqy ffmpeg +RUN apt-get install -qqy yt-dlp +RUN apt-get install -qqy entr + +RUN pip install pipenv + +RUN mkdir -p /home/user +RUN chown 1000:1000 -R /home/user +RUN useradd --uid 1000 --shell /bin/bash --user-group -d /home/user user + +USER 1000 +WORKDIR /src + + + + +CMD ["/bin/bash"] diff --git a/jason78989/project/README.md b/jason78989/project/README.md new file mode 100644 index 00000000..e69de29b diff --git a/jason78989/project/dc_up.bat b/jason78989/project/dc_up.bat new file mode 100644 index 00000000..38874671 --- /dev/null +++ b/jason78989/project/dc_up.bat @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex + +docker compose pull +docker compose build +docker compose up -d +docker compose exec -it test bash diff --git a/jason78989/project/dc_up.sh b/jason78989/project/dc_up.sh new file mode 100644 index 00000000..7a0a5f83 --- /dev/null +++ b/jason78989/project/dc_up.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -ex + +# docker compose pull +# docker compose build +docker compose restart +docker compose up -d +docker compose logs -f +# docker compose exec -it test bash diff --git a/jason78989/project/docker-compose.yml b/jason78989/project/docker-compose.yml new file mode 100644 index 00000000..164b22ca --- /dev/null +++ b/jason78989/project/docker-compose.yml @@ -0,0 +1,19 @@ +services: + test: + build: . + command: ./run.sh + # command: sleep infinity + env_file: ./.env + volumes: + - /etc/localtime:/etc/localtime:ro + - ./src:/src + working_dir: /src + # + # - /run/docker.sock:/run/docker.sock:ro + # restart: always + # environment: + # - TZ=Asia/Hong_Kong + # deploy: + # resources: + # reservations: + # cpus: '0.01' diff --git a/jason78989/project/src/Pipfile b/jason78989/project/src/Pipfile new file mode 100644 index 00000000..099d2354 --- /dev/null +++ b/jason78989/project/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +twscrape = "*" +twspace-dl = "*" + +[dev-packages] + +[requires] +python_version = "3.10" diff --git a/jason78989/project/src/Pipfile.lock b/jason78989/project/src/Pipfile.lock new file mode 100644 index 00000000..d547357c --- /dev/null +++ b/jason78989/project/src/Pipfile.lock @@ -0,0 +1,254 @@ +{ + "_meta": { + "hash": { + "sha256": "be43ee06762c7970e42fb90b35e3e02d7c8cfb7cab55807a89e868e2223eefc6" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.10" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aiosqlite": { + "hashes": [ + "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d", + "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96" + ], + "markers": "python_version >= '3.7'", + "version": "==0.19.0" + }, + "anyio": { + "hashes": [ + "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee", + "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f" + ], + "markers": "python_version >= '3.8'", + "version": "==4.2.0" + }, + "certifi": { + "hashes": [ + "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1", + "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474" + ], + "markers": "python_version >= '3.6'", + "version": "==2023.11.17" + }, + "charset-normalizer": { + "hashes": [ + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.2" + }, + "exceptiongroup": { + "hashes": [ + "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", + "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68" + ], + "markers": "python_version < '3.11'", + "version": "==1.2.0" + }, + "fake-useragent": { + "hashes": [ + "sha256:5426e4015d8ccc5bb25f64d3dfcfd3915eba30ffebd31b86b60dc7a4c5d65528", + "sha256:9acce439ee2c6cf9c3772fa6c200f62dc8d56605063327a4d8c5d0e47f414b85" + ], + "version": "==1.4.0" + }, + "h11": { + "hashes": [ + "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", + "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" + ], + "markers": "python_version >= '3.7'", + "version": "==0.14.0" + }, + "httpcore": { + "hashes": [ + "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7", + "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535" + ], + "markers": "python_version >= '3.8'", + "version": "==1.0.2" + }, + "httpx": { + "hashes": [ + "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf", + "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd" + ], + "markers": "python_version >= '3.8'", + "version": "==0.26.0" + }, + "idna": { + "hashes": [ + "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", + "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f" + ], + "markers": "python_version >= '3.5'", + "version": "==3.6" + }, + "loguru": { + "hashes": [ + "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb", + "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac" + ], + "markers": "python_version >= '3.5'", + "version": "==0.7.2" + }, + "mutagen": { + "hashes": [ + "sha256:719fadef0a978c31b4cf3c956261b3c58b6948b32023078a2117b1de09f0fc99", + "sha256:edd96f50c5907a9539d8e5bba7245f62c9f520aef333d13392a79a4f70aca719" + ], + "markers": "python_version >= '3.7'", + "version": "==1.47.0" + }, + "requests": { + "hashes": [ + "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" + ], + "markers": "python_version >= '3.7'", + "version": "==2.31.0" + }, + "sniffio": { + "hashes": [ + "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101", + "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384" + ], + "markers": "python_version >= '3.7'", + "version": "==1.3.0" + }, + "twscrape": { + "hashes": [ + "sha256:cacb762493a90dab1ff12894e2a436c15b381b2cafc2238a9b0a118eda8cecc3", + "sha256:d3220b5617e6917e3fbd211a3b5260fde46b9ebab5dd86057ac74db454790bc1" + ], + "index": "pypi", + "markers": "python_version >= '3.10'", + "version": "==0.9.0" + }, + "twspace-dl": { + "hashes": [ + "sha256:0be28552a9e0605f579b85671ab8c4a853b9652c91d6e249cacc659aa6217105", + "sha256:3aaf64e4d7e2c5fd6f8a18676bb837664a82c04b42767be56f122e3a7195a0a1" + ], + "index": "pypi", + "markers": "python_version >= '3.8' and python_version < '4.0'", + "version": "==2023.7.24.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783", + "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd" + ], + "markers": "python_version < '3.11'", + "version": "==4.9.0" + }, + "urllib3": { + "hashes": [ + "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3", + "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54" + ], + "markers": "python_version >= '3.8'", + "version": "==2.1.0" + } + }, + "develop": {} +} diff --git a/jason78989/project/src/_downloaded/.gitkeep b/jason78989/project/src/_downloaded/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/jason78989/project/src/discover.py b/jason78989/project/src/discover.py new file mode 100644 index 00000000..3af94b41 --- /dev/null +++ b/jason78989/project/src/discover.py @@ -0,0 +1,67 @@ +import os,sys +from twscrape import API, gather +from twscrape.logger import set_log_level +from pprint import pprint +import json + +TW_USERNAME = os.getenv('TW_USERNAME') +TW_PASSWORD = os.getenv('TW_PASSWORD') +TW_EMAIL = os.getenv('TW_EMAIL') +TW_EMAIL_PASSWORD = os.getenv('TW_EMAIL_PASSWORD') +URL_IGNORE_FILE = './url_ignore.txt' + +MAX_LENGTH = 50 + +async def insertAccount(): + api = API() + await api.pool.add_account(TW_USERNAME, TW_PASSWORD, TW_EMAIL, TW_EMAIL_PASSWORD) + sys.exit(999) + +async def discover(user_login = "LouiscklawLouis"): + api = API() + await api.pool.login_all() + + user = await api.user_by_login(user_login) + print(user.id) + + user_id = user.id + test_list = await gather(api.user_tweets(user_id, limit=5)) + + # from pprint import pprint + # pprint(test_list) + # sys.exit() + + url_list = [] + + for i in range(0, len(test_list)): + for j in range(0, len(test_list[i].links)): + extracted_url = test_list[i].links[j].url + print(f'push into list: {extracted_url}') + url_list.append(extracted_url) + + return url_list + +def read_url_ignore_list(): + url_ignore_list = [] + with open(URL_IGNORE_FILE,'r') as f_url_ignore: + url_ignore_list = list(map(lambda x: x.strip(), f_url_ignore.readlines())) + + return url_ignore_list + + +def update_ignore_list(url_to_ignore): + print("update url ignore list") + + url_ignore_list = read_url_ignore_list() + print(url_ignore_list) + + with open(URL_IGNORE_FILE,'r+') as f_url_ignore: + f_url_ignore.truncate(0) + url_ignore_list.append(url_to_ignore) + + if (len(url_ignore_list )> MAX_LENGTH): + url_ignore_list = url_ignore_list[1:MAX_LENGTH] + f_url_ignore.writelines("\n".join(url_ignore_list)) + +def helloworld(): + print("Helloworld") diff --git a/jason78989/project/src/download/broadcast.py b/jason78989/project/src/download/broadcast.py new file mode 100644 index 00000000..a8d22b02 --- /dev/null +++ b/jason78989/project/src/download/broadcast.py @@ -0,0 +1,20 @@ +import subprocess +import os,sys + +def download_broadcast(resources_url = 'https://twitter.com/i/broadcasts/1OyKAWRYeOgJb'): + cwd = os.path.dirname(__file__) + command = ' '.join(["./download_broadcast.sh",resources_url]) + process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, shell=True) + output, error = process.communicate() + + if process.returncode == 0: + print("Command executed successfully!") + print("Output:\n", output.decode()) + else: + print("Error executing command:", error.decode()) + +def helloworld(): + print("helloworld") + +if __name__ == "__main__": + download() diff --git a/jason78989/project/src/download/discord.py b/jason78989/project/src/download/discord.py new file mode 100644 index 00000000..9361ee00 --- /dev/null +++ b/jason78989/project/src/download/discord.py @@ -0,0 +1,17 @@ +import os,sys +import requests + +def send_message(webhook_url, message): + data = { + 'content': message + } + response = requests.post(webhook_url, json=data) + if response.status_code == 204: + print('Message sent successfully.') + else: + print('Failed to send message.') + +# webhook_url = 'https://discord.com/api/webhooks/1188733135014350858/WMbHoxDuepmiJOifvIzMfxMzBF3AYTOEo9OaEM7QKFIj9sj0SeXnUOQd-rMO6KFMiirl' + +# message = "Hello, Discord!" +# send_message(webhook_url, message) diff --git a/jason78989/project/src/download/download_broadcast.sh b/jason78989/project/src/download/download_broadcast.sh new file mode 100644 index 00000000..af7f1021 --- /dev/null +++ b/jason78989/project/src/download/download_broadcast.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -ex + +rm -rf *.part *.ytdl +sleep 1 + +if pgrep -x "yt-dlp" > /dev/null; then + echo "Process is running." +else + echo "Process is not running." + yt-dlp $1 + # yt-dlp https://twitter.com/i/broadcasts/1OyKAWRYeOgJb +fi + +mv *.mp4 ../_downloaded/ diff --git a/jason78989/project/src/download/download_space.sh b/jason78989/project/src/download/download_space.sh new file mode 100644 index 00000000..5c761972 --- /dev/null +++ b/jason78989/project/src/download/download_space.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex + +# ./download_space.sh https://twitter.com/i/spaces/1LyGBnmdpejGN + +# twspace_dl -v -i $1 -c cookies.txt -u master_url.txt + +# https://x.com/i/spaces/1kvJpvwyBvQKE + +if pgrep -x "twspace_dl" > /dev/null; then + echo "Process is running." +else + echo "Process is not running." + # twspace_dl -v -i https://twitter.com/i/spaces/1BdxYrLqVwNKX -c cookies.txt + twspace_dl -v -i $1 -c cookies.txt +fi + +mv *.m4a ../_downloaded/ diff --git a/jason78989/project/src/download/space.py b/jason78989/project/src/download/space.py new file mode 100644 index 00000000..e3c759ba --- /dev/null +++ b/jason78989/project/src/download/space.py @@ -0,0 +1,17 @@ +import subprocess +import os,sys + +def download_space(space_url="https://twitter.com/i/spaces/1LyGBnmdpejGN" ): + cwd = os.path.dirname(__file__) + command = ' '.join(["./download_space.sh",space_url]) + process = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, shell=True) + output, error = process.communicate() + + if process.returncode == 0: + print("Command executed successfully!") + print("Output:\n", output.decode()) + else: + print("Error executing command:", error.decode()) + +def helloworld(): + print("helloworld") \ No newline at end of file diff --git a/jason78989/project/src/download/test_download_live.sh b/jason78989/project/src/download/test_download_live.sh new file mode 100644 index 00000000..7d485d92 --- /dev/null +++ b/jason78989/project/src/download/test_download_live.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +yt-dlp https://twitter.com/i/broadcasts/1OyKAWRWROgJb diff --git a/jason78989/project/src/download/tmp1tjxl9pd/list.txt b/jason78989/project/src/download/tmp1tjxl9pd/list.txt new file mode 100644 index 00000000..007bbf4a --- /dev/null +++ b/jason78989/project/src/download/tmp1tjxl9pd/list.txt @@ -0,0 +1,2 @@ +file '/src/download/tmp1tjxl9pd/(louis)f-1OyKAWRLDkbJb.m4a' +file '/src/download/tmp1tjxl9pd/(louis)f-1OyKAWRLDkbJb_new.m4a' \ No newline at end of file diff --git a/jason78989/project/src/download/tmpeubq7koa/list.txt b/jason78989/project/src/download/tmpeubq7koa/list.txt new file mode 100644 index 00000000..cbdb5bc3 --- /dev/null +++ b/jason78989/project/src/download/tmpeubq7koa/list.txt @@ -0,0 +1,2 @@ +file '/src/download/tmpeubq7koa/(louis)-1yoKMwgoZeXJQ.m4a' +file '/src/download/tmpeubq7koa/(louis)-1yoKMwgoZeXJQ_new.m4a' \ No newline at end of file diff --git a/jason78989/project/src/entry.sh b/jason78989/project/src/entry.sh new file mode 100644 index 00000000..d4d9deae --- /dev/null +++ b/jason78989/project/src/entry.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -ex + +pipenv sync + +./run.py diff --git a/jason78989/project/src/main.py b/jason78989/project/src/main.py new file mode 100644 index 00000000..676f7017 --- /dev/null +++ b/jason78989/project/src/main.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import os,sys +import asyncio +import datetime +from pprint import pprint + +from download.space import download_space +from download.broadcast import download_broadcast +from discover import discover, read_url_ignore_list, update_ignore_list +from download.discord import send_message + +async def main(): + current_time = datetime.datetime.now() + day_string = current_time.strftime("%Y-%m-%d %H:%M:%S") + print(day_string) + + list_last_run = [] + with open('./last_run.txt','r') as f_last_run: + list_last_run = f_last_run.readlines() + + list_last_run.append(day_string) + with open('./last_run.txt','r+') as f_last_run: + f_last_run.truncate(0) + f_last_run.writelines(list_last_run) + f_last_run.writelines('\n') + + try: + if os.path.exists('./download/cookies.txt'): + pass + else: + raise ValueError("cookies file not found.") + + DISCORD_WEBOOOK_URL = os.getenv('DISCORD_WEBOOOK_URL') + + # await insertAccount() + print('discover new resources url') + url_list = await discover('LouiscklawLouis') + + for url in url_list: + url = url.strip() + url_ignore_list = read_url_ignore_list() + + if (url in url_ignore_list): + print("url already in ignore list, ignore") + else: + if (url.find('spaces')> -1): + try: + space_url = url + print(f'downloading space {space_url}') + + # download the master url + download_space(space_url) + + print("download done") + print('update to ignore list') + + # add master url to ignore list + update_ignore_list(space_url) + + send_message(DISCORD_WEBOOOK_URL,'new space downloaded') + + except Exception as e: + print(e) + + # if (url.find('broadcasts')> -1): + # try: + # broadcast_url = url + # print(f'downloading broadcast {broadcast_url}') + # download_broadcast(broadcast_url) + # print("download done") + # print('update to ignore list') + # update_ignore_list(broadcast_url) + # send_message(DISCORD_WEBOOOK_URL,'new broadcast downloaded') + # except Exception as e: + # print(e) + # pass + + print("done") + + + except Exception as e: + print(e) + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/jason78989/project/src/run.sh b/jason78989/project/src/run.sh new file mode 100644 index 00000000..41da1d90 --- /dev/null +++ b/jason78989/project/src/run.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -ex + +echo "init" +# pipenv run python ./step1-monitor.py + +while true; do + echo "perform check" + pipenv sync + pipenv run python ./main.py 2>&1 + + echo "sleep a while" + sleep 30 +done diff --git a/jason78989/project/src/step1-monitor.py b/jason78989/project/src/step1-monitor.py new file mode 100644 index 00000000..293f7e71 --- /dev/null +++ b/jason78989/project/src/step1-monitor.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +import asyncio +from twscrape import API, gather +from twscrape.logger import set_log_level + +async def main(): + api = API() + # await api.pool.add_account("louiscklaw", "24p69gki", "louiscklaw@gmail.com", "24p69gki") + await api.pool.login_all() + + user_login = "LouiscklawLouis" + user = await api.user_by_login(user_login) + print(user.id) + + user_id = user.id + test_list = await gather(api.user_tweets(user_id, limit=5)) + from pprint import pprint + pprint(test_list[0].links[0].url) + + + # get the link https://twitter.com/i/broadcasts/1DXGyjnAQVeJM + # process to to download + + # get the link + # https://x.com/i/spaces/1LyGBnmdpejGN + # process + # space_links = 'https://x.com/i/spaces/1LyGBnmdpejGN' + + # # new space resources found + # # download space resources + + # print(test_list) + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/jason78989/task1/step1-monitor/result.txt b/jason78989/task1/step1-monitor/result.txt new file mode 100644 index 00000000..3ffbe0d1 --- /dev/null +++ b/jason78989/task1/step1-monitor/result.txt @@ -0,0 +1,157 @@ +4887255340 +[Tweet(id=1739151195732840940, + id_str='1739151195732840940', + url='https://twitter.com/LouiscklawLouis/status/1739151195732840940', + date=datetime.datetime(2023, 12, 25, 5, 8, 51, tzinfo=datetime.timezone.utc), + user=User(id=4887255340, + id_str='4887255340', + url='https://twitter.com/LouiscklawLouis', + username='LouiscklawLouis', + displayname='louis', + rawDescription='', + created=datetime.datetime(2016, 2, 8, 7, 23, 20, tzinfo=datetime.timezone.utc), + followersCount=1, + friendsCount=39, + statusesCount=3, + favouritesCount=0, + listedCount=0, + mediaCount=0, + location='', + profileImageUrl='https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png', + profileBannerUrl=None, + protected=None, + verified=False, + blue=False, + blueType=None, + descriptionLinks=[], + _type='snscrape.modules.twitter.User'), + lang='zxx', + rawContent='https://t.co/a0YX1eAmer', + replyCount=0, + retweetCount=0, + likeCount=0, + quoteCount=0, + conversationId=1739151195732840940, + hashtags=[], + cashtags=[], + mentionedUsers=[], + links=[TextLink(url='https://x.com/i/spaces/1LyGBnmdpejGN', + text='x.com/i/spaces/1LyGB…', + tcourl='https://t.co/a0YX1eAmer')], + viewCount=1, + retweetedTweet=None, + quotedTweet=None, + place=None, + coordinates=None, + inReplyToTweetId=None, + inReplyToUser=None, + source='Twitter for Android', + sourceUrl='http://twitter.com/download/android', + sourceLabel='Twitter for Android', + media=Media(photos=[], videos=[], animated=[]), + _type='snscrape.modules.twitter.Tweet'), + Tweet(id=1739150833206612066, + id_str='1739150833206612066', + url='https://twitter.com/LouiscklawLouis/status/1739150833206612066', + date=datetime.datetime(2023, 12, 25, 5, 7, 25, tzinfo=datetime.timezone.utc), + user=User(id=4887255340, + id_str='4887255340', + url='https://twitter.com/LouiscklawLouis', + username='LouiscklawLouis', + displayname='louis', + rawDescription='', + created=datetime.datetime(2016, 2, 8, 7, 23, 20, tzinfo=datetime.timezone.utc), + followersCount=1, + friendsCount=39, + statusesCount=3, + favouritesCount=0, + listedCount=0, + mediaCount=0, + location='', + profileImageUrl='https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png', + profileBannerUrl=None, + protected=None, + verified=False, + blue=False, + blueType=None, + descriptionLinks=[], + _type='snscrape.modules.twitter.User'), + lang='zxx', + rawContent='https://t.co/DveNOVgyy1', + replyCount=0, + retweetCount=0, + likeCount=0, + quoteCount=0, + conversationId=1739150833206612066, + hashtags=[], + cashtags=[], + mentionedUsers=[], + links=[TextLink(url='https://twitter.com/i/broadcasts/1lPKqbjXavEGb', + text='twitter.com/i/broadcasts/1…', + tcourl='https://t.co/DveNOVgyy1')], + viewCount=None, + retweetedTweet=None, + quotedTweet=None, + place=None, + coordinates=None, + inReplyToTweetId=None, + inReplyToUser=None, + source='Twitter for Android', + sourceUrl='http://twitter.com/download/android', + sourceLabel='Twitter for Android', + media=Media(photos=[], videos=[], animated=[]), + _type='snscrape.modules.twitter.Tweet'), + Tweet(id=1738999403795607942, + id_str='1738999403795607942', + url='https://twitter.com/LouiscklawLouis/status/1738999403795607942', + date=datetime.datetime(2023, 12, 24, 19, 5, 41, tzinfo=datetime.timezone.utc), + user=User(id=4887255340, + id_str='4887255340', + url='https://twitter.com/LouiscklawLouis', + username='LouiscklawLouis', + displayname='louis', + rawDescription='', + created=datetime.datetime(2016, 2, 8, 7, 23, 20, tzinfo=datetime.timezone.utc), + followersCount=1, + friendsCount=39, + statusesCount=3, + favouritesCount=0, + listedCount=0, + mediaCount=0, + location='', + profileImageUrl='https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png', + profileBannerUrl=None, + protected=None, + verified=False, + blue=False, + blueType=None, + descriptionLinks=[], + _type='snscrape.modules.twitter.User'), + lang='zxx', + rawContent='https://t.co/xGQZandc6E', + replyCount=0, + retweetCount=0, + likeCount=0, + quoteCount=0, + conversationId=1738999403795607942, + hashtags=[], + cashtags=[], + mentionedUsers=[], + links=[TextLink(url='https://twitter.com/i/broadcasts/1DXGyjnAQVeJM', + text='twitter.com/i/broadcasts/1…', + tcourl='https://t.co/xGQZandc6E')], + viewCount=2, + retweetedTweet=None, + quotedTweet=None, + place=None, + coordinates=None, + inReplyToTweetId=None, + inReplyToUser=None, + source='Twitter for Android', + sourceUrl='http://twitter.com/download/android', + sourceLabel='Twitter for Android', + media=Media(photos=[], videos=[], animated=[]), + _type='snscrape.modules.twitter.Tweet')] diff --git a/jason78989/task1/step1-monitor/test.py b/jason78989/task1/step1-monitor/test.py new file mode 100644 index 00000000..7d3cd91a --- /dev/null +++ b/jason78989/task1/step1-monitor/test.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python + +import asyncio +from twscrape import API, gather +from twscrape.logger import set_log_level + +async def main(): + api = API() + # await api.pool.add_account("louiscklaw", "24p69gki", "louiscklaw@gmail.com", "24p69gki") + await api.pool.login_all() + + user_login = "LouiscklawLouis" + user = await api.user_by_login(user_login) + print(user.id) + + user_id = user.id + test_list = await gather(api.user_tweets(user_id, limit=5)) + from pprint import pprint + pprint(test_list) + + # get the link https://twitter.com/i/broadcasts/1DXGyjnAQVeJM + # process to to download + + # get the link + # https://x.com/i/spaces/1LyGBnmdpejGN + # process + + # print(test_list) + +if __name__ == "__main__": + asyncio.run(main()) + diff --git a/jason78989/task1/step1-monitor/test.sh b/jason78989/task1/step1-monitor/test.sh new file mode 100644 index 00000000..c95b55e1 --- /dev/null +++ b/jason78989/task1/step1-monitor/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex + +# twscrape louiscklaw@gmail.com 24p69gki +# twscrape add_accounts ./accounts.txt louiscklaw:24p69gki:louiscklaw@gmail.com:24p69gki + +twscrape add_accounts ./accounts.txt username:password:email:123321 \ No newline at end of file diff --git a/jason78989/task1/step2-download-space/download_space.sh b/jason78989/task1/step2-download-space/download_space.sh new file mode 100644 index 00000000..e7f44269 --- /dev/null +++ b/jason78989/task1/step2-download-space/download_space.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -ex + +# https://twitter.com/i/spaces/1LyxBnmNkObxN + +# twspace_dl -v -i https://twitter.com/i/spaces/1LyxBnmNkObxN -c cookies.txt + +# https://twitter.com/i/spaces/1LyGBnmdpejGN/peek +twspace_dl -v -i https://twitter.com/i/spaces/1LyGBnmdpejGN -c cookies.txt + + + +# pip install git+https://github.com/rly0nheart/twitter-video-downloader \ No newline at end of file diff --git a/jason78989/task1/step3-download-broadcast/test.sh b/jason78989/task1/step3-download-broadcast/test.sh new file mode 100644 index 00000000..be20ede8 --- /dev/null +++ b/jason78989/task1/step3-download-broadcast/test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +yt-dlp https://twitter.com/i/broadcasts/1OyKAWRYeOgJb diff --git a/jason78989/task1/step4-send-alert/send-discord.py b/jason78989/task1/step4-send-alert/send-discord.py new file mode 100644 index 00000000..4ac6576f --- /dev/null +++ b/jason78989/task1/step4-send-alert/send-discord.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python + +import os,sys +from pprint import pprint +import requests + +def send_message(webhook_url, message): + data = { + 'content': message + } + response = requests.post(webhook_url, json=data) + if response.status_code == 204: + print('Message sent successfully.') + else: + print('Failed to send message.') + +webhook_url = 'https://discord.com/api/webhooks/1188733135014350858/WMbHoxDuepmiJOifvIzMfxMzBF3AYTOEo9OaEM7QKFIj9sj0SeXnUOQd-rMO6KFMiirl' + +message = "Hello, Discord!" +send_message(webhook_url, message)