# HG changeset patch # User Ludovic Chabant # Date 1684687331 25200 # Node ID c5f73ebb43a54d4df25096b74cd5da7144a6aef8 # Parent 0f98784bcc40e6ec0ef98140b7b7d1870c1c12e8 Replace python-twitter with tweepy to use Twitter's V2 API diff -r 0f98784bcc40 -r c5f73ebb43a5 Pipfile --- a/Pipfile Sun May 21 09:40:00 2023 -0700 +++ b/Pipfile Sun May 21 09:42:11 2023 -0700 @@ -7,14 +7,14 @@ [packages] -"mf2py" = "*" "mastodon.py" = "*" coloredlogs = "*" -"mf2util" = "*" dateparser = "*" +mf2py = "*" +mf2util = "*" python-dateutil = "*" -python-twitter = "*" ronkyuu = "*" +tweepy = "*" [dev-packages] diff -r 0f98784bcc40 -r c5f73ebb43a5 Pipfile.lock --- a/Pipfile.lock Sun May 21 09:40:00 2023 -0700 +++ b/Pipfile.lock Sun May 21 09:42:11 2023 -0700 @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2107dacb7d1214bf5e04f786633d7e1b3e8315a9e4997c49129edb8ff58cb4ec" + "sha256": "e445c8ccfe63fdfd308f3a7bd5b71cb2ca4ba2ce309fc09310e4ae69bbf7c1bd" }, "pipfile-spec": 6, "requires": {}, @@ -126,6 +126,14 @@ "index": "pypi", "version": "==15.0.1" }, + "dateparser": { + "hashes": [ + "sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f", + "sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3" + ], + "index": "pypi", + "version": "==1.1.8" + }, "decorator": { "hashes": [ "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", @@ -134,13 +142,6 @@ "markers": "python_version >= '3.5'", "version": "==5.1.1" }, - "future": { - "hashes": [ - "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.18.3" - }, "html5lib": { "hashes": [ "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", @@ -211,13 +212,106 @@ "markers": "platform_system != 'Windows'", "version": "==0.4.27" }, - "python-twitter": { + "pytz": { + "hashes": [ + "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", + "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" + ], + "version": "==2023.3" + }, + "regex": { "hashes": [ - "sha256:45855742f1095aa0c8c57b2983eee3b6b7f527462b50a2fa8437a8b398544d90", - "sha256:4a420a6cb6ee9d0c8da457c8a8573f709c2ff2e1a7542e2d38807ebbfe8ebd1d" + "sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff", + "sha256:0a69cf0c00c4d4a929c6c7717fd918414cab0d6132a49a6d8fc3ded1988ed2ea", + "sha256:0bbd5dcb19603ab8d2781fac60114fb89aee8494f4505ae7ad141a3314abb1f9", + "sha256:10250a093741ec7bf74bcd2039e697f519b028518f605ff2aa7ac1e9c9f97423", + "sha256:10374c84ee58c44575b667310d5bbfa89fb2e64e52349720a0182c0017512f6c", + "sha256:1189fbbb21e2c117fda5303653b61905aeeeea23de4a94d400b0487eb16d2d60", + "sha256:1307aa4daa1cbb23823d8238e1f61292fd07e4e5d8d38a6efff00b67a7cdb764", + "sha256:144b5b017646b5a9392a5554a1e5db0000ae637be4971c9747566775fc96e1b2", + "sha256:171c52e320fe29260da550d81c6b99f6f8402450dc7777ef5ced2e848f3b6f8f", + "sha256:18196c16a584619c7c1d843497c069955d7629ad4a3fdee240eb347f4a2c9dbe", + "sha256:18f05d14f14a812fe9723f13afafefe6b74ca042d99f8884e62dbd34dcccf3e2", + "sha256:1ecf3dcff71f0c0fe3e555201cbe749fa66aae8d18f80d2cc4de8e66df37390a", + "sha256:21e90a288e6ba4bf44c25c6a946cb9b0f00b73044d74308b5e0afd190338297c", + "sha256:23d86ad2121b3c4fc78c58f95e19173790e22ac05996df69b84e12da5816cb17", + "sha256:256f7f4c6ba145f62f7a441a003c94b8b1af78cee2cccacfc1e835f93bc09426", + "sha256:290fd35219486dfbc00b0de72f455ecdd63e59b528991a6aec9fdfc0ce85672e", + "sha256:2e9c4f778514a560a9c9aa8e5538bee759b55f6c1dcd35613ad72523fd9175b8", + "sha256:338994d3d4ca4cf12f09822e025731a5bdd3a37aaa571fa52659e85ca793fb67", + "sha256:33d430a23b661629661f1fe8395be2004006bc792bb9fc7c53911d661b69dd7e", + "sha256:385992d5ecf1a93cb85adff2f73e0402dd9ac29b71b7006d342cc920816e6f32", + "sha256:3d45864693351c15531f7e76f545ec35000d50848daa833cead96edae1665559", + "sha256:40005cbd383438aecf715a7b47fe1e3dcbc889a36461ed416bdec07e0ef1db66", + "sha256:4035d6945cb961c90c3e1c1ca2feb526175bcfed44dfb1cc77db4fdced060d3e", + "sha256:445d6f4fc3bd9fc2bf0416164454f90acab8858cd5a041403d7a11e3356980e8", + "sha256:48c9ec56579d4ba1c88f42302194b8ae2350265cb60c64b7b9a88dcb7fbde309", + "sha256:4a5059bd585e9e9504ef9c07e4bc15b0a621ba20504388875d66b8b30a5c4d18", + "sha256:4a6e4b0e0531223f53bad07ddf733af490ba2b8367f62342b92b39b29f72735a", + "sha256:4b870b6f632fc74941cadc2a0f3064ed8409e6f8ee226cdfd2a85ae50473aa94", + "sha256:50fd2d9b36938d4dcecbd684777dd12a407add4f9f934f235c66372e630772b0", + "sha256:53e22e4460f0245b468ee645156a4f84d0fc35a12d9ba79bd7d79bdcd2f9629d", + "sha256:586a011f77f8a2da4b888774174cd266e69e917a67ba072c7fc0e91878178a80", + "sha256:59597cd6315d3439ed4b074febe84a439c33928dd34396941b4d377692eca810", + "sha256:59e4b729eae1a0919f9e4c0fc635fbcc9db59c74ad98d684f4877be3d2607dd6", + "sha256:5a0f874ee8c0bc820e649c900243c6d1e6dc435b81da1492046716f14f1a2a96", + "sha256:5ac2b7d341dc1bd102be849d6dd33b09701223a851105b2754339e390be0627a", + "sha256:5e3f4468b8c6fd2fd33c218bbd0a1559e6a6fcf185af8bb0cc43f3b5bfb7d636", + "sha256:6164d4e2a82f9ebd7752a06bd6c504791bedc6418c0196cd0a23afb7f3e12b2d", + "sha256:6893544e06bae009916a5658ce7207e26ed17385149f35a3125f5259951f1bbe", + "sha256:690a17db524ee6ac4a27efc5406530dd90e7a7a69d8360235323d0e5dafb8f5b", + "sha256:6b8d0c153f07a953636b9cdb3011b733cadd4178123ef728ccc4d5969e67f3c2", + "sha256:72a28979cc667e5f82ef433db009184e7ac277844eea0f7f4d254b789517941d", + "sha256:72aa4746993a28c841e05889f3f1b1e5d14df8d3daa157d6001a34c98102b393", + "sha256:732176f5427e72fa2325b05c58ad0b45af341c459910d766f814b0584ac1f9ac", + "sha256:7918a1b83dd70dc04ab5ed24c78ae833ae8ea228cef84e08597c408286edc926", + "sha256:7923470d6056a9590247ff729c05e8e0f06bbd4efa6569c916943cb2d9b68b91", + "sha256:7d76a8a1fc9da08296462a18f16620ba73bcbf5909e42383b253ef34d9d5141e", + "sha256:811040d7f3dd9c55eb0d8b00b5dcb7fd9ae1761c454f444fd9f37fe5ec57143a", + "sha256:821a88b878b6589c5068f4cc2cfeb2c64e343a196bc9d7ac68ea8c2a776acd46", + "sha256:84397d3f750d153ebd7f958efaa92b45fea170200e2df5e0e1fd4d85b7e3f58a", + "sha256:844671c9c1150fcdac46d43198364034b961bd520f2c4fdaabfc7c7d7138a2dd", + "sha256:890a09cb0a62198bff92eda98b2b507305dd3abf974778bae3287f98b48907d3", + "sha256:8f08276466fedb9e36e5193a96cb944928301152879ec20c2d723d1031cd4ddd", + "sha256:8f5e06df94fff8c4c85f98c6487f6636848e1dc85ce17ab7d1931df4a081f657", + "sha256:921473a93bcea4d00295799ab929522fc650e85c6b9f27ae1e6bb32a790ea7d3", + "sha256:941b3f1b2392f0bcd6abf1bc7a322787d6db4e7457be6d1ffd3a693426a755f2", + "sha256:9b320677521aabf666cdd6e99baee4fb5ac3996349c3b7f8e7c4eee1c00dfe3a", + "sha256:9c3efee9bb53cbe7b285760c81f28ac80dc15fa48b5fe7e58b52752e642553f1", + "sha256:9fda3e50abad8d0f48df621cf75adc73c63f7243cbe0e3b2171392b445401550", + "sha256:a4c5da39bca4f7979eefcbb36efea04471cd68db2d38fcbb4ee2c6d440699833", + "sha256:a56c18f21ac98209da9c54ae3ebb3b6f6e772038681d6cb43b8d53da3b09ee81", + "sha256:a623564d810e7a953ff1357f7799c14bc9beeab699aacc8b7ab7822da1e952b8", + "sha256:a8906669b03c63266b6a7693d1f487b02647beb12adea20f8840c1a087e2dfb5", + "sha256:a99757ad7fe5c8a2bb44829fc57ced11253e10f462233c1255fe03888e06bc19", + "sha256:aa7d032c1d84726aa9edeb6accf079b4caa87151ca9fabacef31fa028186c66d", + "sha256:aad5524c2aedaf9aa14ef1bc9327f8abd915699dea457d339bebbe2f0d218f86", + "sha256:afb1c70ec1e594a547f38ad6bf5e3d60304ce7539e677c1429eebab115bce56e", + "sha256:b6365703e8cf1644b82104cdd05270d1a9f043119a168d66c55684b1b557d008", + "sha256:b8b942d8b3ce765dbc3b1dad0a944712a89b5de290ce8f72681e22b3c55f3cc8", + "sha256:ba73a14e9c8f9ac409863543cde3290dba39098fc261f717dc337ea72d3ebad2", + "sha256:bd7b68fd2e79d59d86dcbc1ccd6e2ca09c505343445daaa4e07f43c8a9cc34da", + "sha256:bd966475e963122ee0a7118ec9024388c602d12ac72860f6eea119a3928be053", + "sha256:c2ce65bdeaf0a386bb3b533a28de3994e8e13b464ac15e1e67e4603dd88787fa", + "sha256:c64d5abe91a3dfe5ff250c6bb267ef00dbc01501518225b45a5f9def458f31fb", + "sha256:c8c143a65ce3ca42e54d8e6fcaf465b6b672ed1c6c90022794a802fb93105d22", + "sha256:cd46f30e758629c3ee91713529cfbe107ac50d27110fdcc326a42ce2acf4dafc", + "sha256:ced02e3bd55e16e89c08bbc8128cff0884d96e7f7a5633d3dc366b6d95fcd1d6", + "sha256:cf123225945aa58b3057d0fba67e8061c62d14cc8a4202630f8057df70189051", + "sha256:d19e57f888b00cd04fc38f5e18d0efbd91ccba2d45039453ab2236e6eec48d4d", + "sha256:d1cbe6b5be3b9b698d8cc4ee4dee7e017ad655e83361cd0ea8e653d65e469468", + "sha256:db09e6c18977a33fea26fe67b7a842f706c67cf8bda1450974d0ae0dd63570df", + "sha256:de2f780c3242ea114dd01f84848655356af4dd561501896c751d7b885ea6d3a1", + "sha256:e2205a81f815b5bb17e46e74cc946c575b484e5f0acfcb805fb252d67e22938d", + "sha256:e645c757183ee0e13f0bbe56508598e2d9cd42b8abc6c0599d53b0d0b8dd1479", + "sha256:f2910502f718828cecc8beff004917dcf577fc5f8f5dd40ffb1ea7612124547b", + "sha256:f764e4dfafa288e2eba21231f455d209f4709436baeebb05bdecfb5d8ddc3d35", + "sha256:f83fe9e10f9d0b6cf580564d4d23845b9d692e4c91bd8be57733958e4c602956", + "sha256:fb2b495dd94b02de8215625948132cc2ea360ae84fe6634cd19b6567709c8ae2", + "sha256:fee0016cc35a8a91e8cc9312ab26a6fe638d484131a7afa79e1ce6165328a135" ], - "index": "pypi", - "version": "==3.5" + "markers": "python_version >= '3.6'", + "version": "==2023.5.5" }, "requests": { "hashes": [ @@ -259,6 +353,22 @@ "markers": "python_version >= '3.7'", "version": "==2.4.1" }, + "tweepy": { + "hashes": [ + "sha256:1f9f1707d6972de6cff6c5fd90dfe6a449cd2e0d70bd40043ffab01e07a06c8c", + "sha256:db6d3844ccc0c6d27f339f12ba8acc89912a961da513c1ae50fa2be502a56afb" + ], + "index": "pypi", + "version": "==4.14.0" + }, + "tzlocal": { + "hashes": [ + "sha256:46eb99ad4bdb71f3f72b7d24f4267753e240944ecfc16f25d2719ba89827a803", + "sha256:f3596e180296aaf2dbd97d124fe76ae3a0e3d32b258447de7b939b3fd4be992f" + ], + "markers": "python_version >= '3.7'", + "version": "==5.0.1" + }, "urllib3": { "hashes": [ "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc", @@ -326,11 +436,11 @@ }, "setuptools": { "hashes": [ - "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b", - "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990" + "sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f", + "sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102" ], "markers": "python_version >= '3.7'", - "version": "==67.7.2" + "version": "==67.8.0" }, "setuptools-scm": { "hashes": [ diff -r 0f98784bcc40 -r c5f73ebb43a5 requirements.txt --- a/requirements.txt Sun May 21 09:40:00 2023 -0700 +++ b/requirements.txt Sun May 21 09:42:11 2023 -0700 @@ -4,8 +4,8 @@ certifi==2023.5.7 charset-normalizer==3.1.0 coloredlogs==15.0.1 +dateparser==1.1.8 decorator==5.1.1 -future==0.18.3 html5lib==1.1 humanfriendly==10.0 idna==3.4 @@ -15,11 +15,14 @@ oauthlib==3.2.2 python-dateutil==2.8.2 python-magic==0.4.27 -python-twitter==3.5 +pytz==2023.3 +regex==2023.5.5 requests==2.30.0 requests-oauthlib==1.3.1 ronkyuu==0.9 six==1.16.0 soupsieve==2.4.1 +tweepy==4.14.0 +tzlocal==5.0.1 urllib3==2.0.2 webencodings==0.5.1 diff -r 0f98784bcc40 -r c5f73ebb43a5 silorider/silos/twitter.py --- a/silorider/silos/twitter.py Sun May 21 09:40:00 2023 -0700 +++ b/silorider/silos/twitter.py Sun May 21 09:42:11 2023 -0700 @@ -1,7 +1,8 @@ +import os.path import logging -import twitter +import tweepy import urllib.parse -from .base import Silo +from .base import Silo, upload_silo_media from ..format import UrlFlattener from ..parse import strip_img_alt @@ -9,9 +10,32 @@ logger = logging.getLogger(__name__) +class _CompositeClient: + def __init__(self, + consumer_key, consumer_secret, + access_token_key, access_token_secret): + self.v2 = tweepy.Client( + None, # using OAuth v1 + consumer_key=consumer_key, + consumer_secret=consumer_secret, + access_token=access_token_key, + access_token_secret=access_token_secret) + + auth_v1 = tweepy.OAuth1UserHandler( + consumer_key, consumer_secret, + access_token_key, access_token_secret) + self.v1 = tweepy.API(auth_v1) + + def create_tweet(self, *args, **kwargs): + return self.v2.create_tweet(*args, **kwargs) + + def simple_upload(self, *args, **kwargs): + return self.v1.simple_upload(*args, **kwargs) + + class TwitterSilo(Silo): SILO_TYPE = 'twitter' - _CLIENT_CLASS = twitter.Api + _CLIENT_CLASS = _CompositeClient def __init__(self, ctx): super().__init__(ctx) @@ -75,10 +99,10 @@ if not tweettxt: raise Exception("Can't find any content to use for the tweet!") + media_ids = upload_silo_media(entry, 'photo', self._media_callback) + logger.debug("Posting tweet: %s" % tweettxt) - media_urls = entry.get('photo', [], force_list=True) - media_urls = strip_img_alt(media_urls) - self.client.PostUpdate(tweettxt, media=media_urls) + self.client.create_tweet(text=tweettxt, media_ids=media_ids) def dryRunPostEntry(self, entry, ctx): tweettxt = self.formatEntry(entry, limit=280, @@ -90,6 +114,14 @@ if media_urls: logger.info("...with photos: %s" % str(media_urls)) + def _media_callback(self, tmpfile, mt, url, desc): + url_parsed = urllib.parse.urlparse(url) + fname = os.path.basename(url_parsed.path) + with open(tmpfile, 'rb') as tmpfp: + logger.debug("Uploading %s to twitter" % fname) + media = self.client.simple_upload(fname, file=tmpfp) + return media.media_id + TWITTER_NETLOCS = ['twitter.com', 'www.twitter.com'] diff -r 0f98784bcc40 -r c5f73ebb43a5 tests/test_silos_twitter.py --- a/tests/test_silos_twitter.py Sun May 21 09:40:00 2023 -0700 +++ b/tests/test_silos_twitter.py Sun May 21 09:42:11 2023 -0700 @@ -1,4 +1,5 @@ import pytest +from .mockutil import mock_urllib def test_one_article(cli, feedutil, tweetmock): @@ -67,7 +68,12 @@ cli.setFeedConfig('feed', feed) tweetmock.installTokens(cli, 'test') - ctx, _ = cli.run('process') + with monkeypatch.context() as m: + import silorider.silos.twitter + mock_urllib(m) + m.setattr(silorider.silos.twitter.TwitterSilo, '_media_callback', + _patched_media_callback) + ctx, _ = cli.run('process') assert ctx.cache.wasPosted('test', '/01234.html') toot = ctx.silos[0].client.tweets[0] @@ -88,7 +94,12 @@ cli.setFeedConfig('feed', feed) tweetmock.installTokens(cli, 'test') - ctx, _ = cli.run('process') + with monkeypatch.context() as m: + import silorider.silos.twitter + mock_urllib(m) + m.setattr(silorider.silos.twitter.TwitterSilo, '_media_callback', + _patched_media_callback) + ctx, _ = cli.run('process') assert ctx.cache.wasPosted('test', '/01234.html') toot = ctx.silos[0].client.tweets[0] @@ -98,10 +109,10 @@ def test_micropost_with_long_text_and_link(cli, feedutil, tweetmock, monkeypatch): feed = cli.createTempFeed(feedutil.makeFeed( - """
-

This a pretty long text that has a link in it :) We want to make sure it gets to the limit of what Twitter allows, so that we can test there won't be any off-by-one errors in measurements. Here is a link to Python's textwrap module, which is appropriate!!!

-
- permalink""" + """
+

This a pretty long text that has a link in it :) We want to make sure it gets to the limit of what Twitter allows, so that we can test there won't be any off-by-one errors in measurements. Here is a link to Python's textwrap module, which is appropriate!!!

+
+ permalink""" )) cli.appendSiloConfig('test', 'twitter', url='/blah') @@ -112,15 +123,15 @@ assert ctx.cache.wasPosted('test', '/01234.html') toot = ctx.silos[0].client.tweets[0] assert toot == ("This a pretty long text that has a link in it :) We want to make sure it gets to the limit of what Twitter allows, so that we can test there won't be any off-by-one errors in measurements. Here is a link to Python's textwrap module, which is appropriate!!! https://docs.python.org/3/library/textwrap.html", - []) + []) def test_micropost_with_too_long_text_and_link_1(cli, feedutil, tweetmock, monkeypatch): feed = cli.createTempFeed(feedutil.makeFeed( - """
-

This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a short word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Trying to get to the limit. Almost here yes

-
- permalink""" + """
+

This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a short word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Trying to get to the limit. Almost here yes

+
+ permalink""" )) cli.appendSiloConfig('test', 'twitter', url='/blah') @@ -131,15 +142,15 @@ assert ctx.cache.wasPosted('test', '/01234.html') toot = ctx.silos[0].client.tweets[0] assert toot == ("This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a short word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Trying to get to the limit. Almost here... /01234.html", - []) + []) def test_micropost_with_too_long_text_and_link_2(cli, feedutil, tweetmock, monkeypatch): feed = cli.createTempFeed(feedutil.makeFeed( - """
-

This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a loooooong word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Our long word is: califragilisticastuff

-
- permalink""" + """
+

This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a loooooong word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Our long word is: califragilisticastuff

+
+ permalink""" )) cli.appendSiloConfig('test', 'twitter', url='/blah') @@ -150,7 +161,11 @@ assert ctx.cache.wasPosted('test', '/01234.html') toot = ctx.silos[0].client.tweets[0] assert toot == ("This time we have a text that's slightly too long, with a link here. We'll be one character too long, with a loooooong word at the end to test the shortening algorithm. Otherwise, don't worry about it. Blah blah blah. Our long word is:... /01234.html", - []) + []) + + +def _patched_media_callback(self, tmpfile, mt, url, desc): + return self.client.simple_upload(url) @pytest.fixture(scope='session') @@ -169,9 +184,23 @@ assert access_token_secret == 'TEST_ACCESS_SECRET' self.tweets = [] + self.media = [] - def PostUpdate(self, tweet, media=None): - self.tweets.append((tweet, media)) + def create_tweet(self, text, media_ids=None): + media_names = [] + if media_ids: + for mid in media_ids: + assert(self.media[mid] is not None) + media_names.append(self.media[mid]) + self.media[mid] = None + assert all([m is None for m in self.media]) + + self.tweets.append((text, media_names)) + self.media = [] + + def simple_upload(self, fname, file=None): + self.media.append(fname) + return len(self.media) - 1 class TwitterMockUtil: