Mercurial > silorider
annotate tests/test_silos_mastodon.py @ 27:c898b4df0f29
Use context for html stripping, with support for custom URL sizes
For instance in Twitter URLs are 23 characters long since they use their
own URL shortening service. Without taking this into account, post lengths
would not be calculated correctly.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Wed, 19 Apr 2023 12:46:58 -0700 |
| parents | a921cc2306bc |
| children | 53de06c2f97d |
| rev | line source |
|---|---|
| 0 | 1 import pytest |
| 2 | 2 from .mockutil import mock_urllib |
| 0 | 3 |
| 4 | |
| 5 def test_one_article(cli, feedutil, mastmock): | |
| 6 feed = cli.createTempFeed(feedutil.makeFeed( | |
| 7 """<h1 class="p-name">A new article</h1> | |
| 8 <div class="e-content"> | |
| 9 <p>This is the text of the article.</p> | |
| 10 <p>It has 2 paragraphs.</p> | |
| 11 </div> | |
| 12 <a class="u-url" href="https://example.org/a-new-article">permalink</a>""" | |
| 13 )) | |
| 14 | |
| 15 cli.appendSiloConfig('test', 'mastodon', url='/blah') | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
16 cli.setFeedConfig('feed', feed) |
| 0 | 17 mastmock.installTokens(cli, 'test') |
| 18 | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
19 ctx, _ = cli.run('process') |
| 0 | 20 assert ctx.cache.wasPosted('test', 'https://example.org/a-new-article') |
| 21 toot = ctx.silos[0].client.toots[0] | |
| 22 assert toot == ('A new article https://example.org/a-new-article', | |
| 23 None, 'public') | |
| 24 | |
| 25 | |
| 26 def test_one_micropost(cli, feedutil, mastmock): | |
| 27 feed = cli.createTempFeed(feedutil.makeFeed( | |
| 28 """<p class="p-name">This is a quick update.</p> | |
| 29 <a class="u-url" href="/01234.html">permalink</a>""" | |
| 30 )) | |
| 31 | |
| 32 cli.appendSiloConfig('test', 'mastodon', url='/blah') | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
33 cli.setFeedConfig('feed', feed) |
| 0 | 34 mastmock.installTokens(cli, 'test') |
| 35 | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
36 ctx, _ = cli.run('process') |
| 0 | 37 assert ctx.cache.wasPosted('test', '/01234.html') |
| 38 toot = ctx.silos[0].client.toots[0] | |
| 39 assert toot == ("This is a quick update.", None, 'public') | |
| 40 | |
| 41 | |
| 42 def test_one_micropost_with_one_photo(cli, feedutil, mastmock, monkeypatch): | |
| 43 feed = cli.createTempFeed(feedutil.makeFeed( | |
| 44 """<p class="p-name">This is a quick photo update.</p> | |
| 45 <div> | |
| 46 <a class="u-photo" href="/fullimg.jpg"><img src="/thumbimg.jpg"/></a> | |
| 47 </div> | |
| 48 <a class="u-url" href="/01234.html">permalink</a>""" | |
| 49 )) | |
| 50 | |
| 51 cli.appendSiloConfig('test', 'mastodon', url='/blah') | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
52 cli.setFeedConfig('feed', feed) |
| 0 | 53 mastmock.installTokens(cli, 'test') |
| 54 | |
| 55 with monkeypatch.context() as m: | |
| 2 | 56 import silorider.silos.mastodon |
| 57 mock_urllib(m) | |
| 58 m.setattr(silorider.silos.mastodon.MastodonSilo, '_media_callback', | |
| 59 _patched_media_callback) | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
60 ctx, _ = cli.run('process') |
| 2 | 61 |
| 0 | 62 assert ctx.cache.wasPosted('test', '/01234.html') |
| 63 media = ctx.silos[0].client.media[0] | |
| 2 | 64 assert media == ('/retrieved/fullimg.jpg', 'image/jpeg', 1) |
| 0 | 65 toot = ctx.silos[0].client.toots[0] |
| 66 assert toot == ("This is a quick photo update.", [1], 'public') | |
| 67 | |
| 68 | |
| 69 def test_one_micropost_with_two_photos(cli, feedutil, mastmock, monkeypatch): | |
| 70 feed = cli.createTempFeed(feedutil.makeFeed( | |
| 71 """<p class="p-name">This is a photo update with 2 photos.</p> | |
| 72 <div> | |
| 73 <a class="u-photo" href="/fullimg1.jpg"><img src="/thumbimg1.jpg"/></a> | |
| 74 <a class="u-photo" href="/fullimg2.jpg"><img src="/thumbimg2.jpg"/></a> | |
| 75 </div> | |
| 76 <a class="u-url" href="/01234.html">permalink</a>""" | |
| 77 )) | |
| 78 | |
| 79 cli.appendSiloConfig('test', 'mastodon', url='/blah') | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
80 cli.setFeedConfig('feed', feed) |
| 0 | 81 mastmock.installTokens(cli, 'test') |
| 82 | |
| 83 with monkeypatch.context() as m: | |
| 2 | 84 import silorider.silos.mastodon |
| 85 mock_urllib(m) | |
| 86 m.setattr(silorider.silos.mastodon.MastodonSilo, '_media_callback', | |
| 87 _patched_media_callback) | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
88 ctx, _ = cli.run('process') |
| 2 | 89 |
| 0 | 90 assert ctx.cache.wasPosted('test', '/01234.html') |
| 91 media = ctx.silos[0].client.media[0] | |
| 2 | 92 assert media == ('/retrieved/fullimg1.jpg', 'image/jpeg', 1) |
| 0 | 93 media = ctx.silos[0].client.media[1] |
| 2 | 94 assert media == ('/retrieved/fullimg2.jpg', 'image/jpeg', 2) |
| 0 | 95 toot = ctx.silos[0].client.toots[0] |
| 96 assert toot == ("This is a photo update with 2 photos.", [1, 2], 'public') | |
| 97 | |
| 98 | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
99 def test_one_micropost_with_links(cli, feedutil, mastmock): |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
100 cli.appendSiloConfig('test', 'mastodon', url='/blah') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
101 mastmock.installTokens(cli, 'test') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
102 |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
103 feed = cli.createTempFeed(feedutil.makeFeed( |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
104 """<p class="p-name">This is a link: http://example.org/blah</p> |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
105 <a class="u-url" href="/01234.html">permalink</a>""")) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
106 cli.setFeedConfig('feed', feed) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
107 ctx, _ = cli.run('process') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
108 toot = ctx.silos[0].client.toots[0] |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
109 assert toot == ("This is a link: http://example.org/blah", None, 'public') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
110 |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
111 feed = cli.createTempFeed(feedutil.makeFeed( |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
112 """<p class="e-content">This is another link: <a href="http://example.org/blah">http://example.org/blah</a></p> |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
113 <a class="u-uri" href="/01234.html">permalink</a>""")) # NOQA |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
114 cli.setFeedConfig('feed', feed) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
115 ctx, _ = cli.run('process') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
116 toot = ctx.silos[0].client.toots[0] |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
117 assert toot == ("This is another link: http://example.org/blah", None, 'public') # NOQA |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
118 |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
119 feed = cli.createTempFeed(feedutil.makeFeed( |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
120 """<p class="e-content">This is yet <a href="http://example.org/blah">another link</a></p> |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
121 <a class="u-uri" href="/01234.html">permalink</a>""")) # NOQA |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
122 cli.setFeedConfig('feed', feed) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
123 ctx, _ = cli.run('process') |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
124 toot = ctx.silos[0].client.toots[0] |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
125 assert toot == ("This is yet another link http://example.org/blah", None, 'public') # NOQA |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
126 |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
2
diff
changeset
|
127 |
| 2 | 128 def _patched_media_callback(self, tmpfile, mt): |
| 129 return self.client.media_post(tmpfile, mt) | |
| 0 | 130 |
| 131 | |
| 132 @pytest.fixture(scope='session') | |
| 133 def mastmock(): | |
| 134 from silorider.silos.mastodon import MastodonSilo | |
| 135 MastodonSilo._CLIENT_CLASS = MastodonMock | |
| 136 return MastodonMockUtil() | |
| 137 | |
| 138 | |
| 139 class MastodonMock: | |
| 140 @staticmethod | |
| 141 def create_app(app_name, scopes, api_base_url): | |
| 142 return ('TEST_CLIENT_ID', 'TEST_CLIENT_SECRET') | |
| 143 | |
| 144 def __init__(self, client_id, client_secret, access_token, api_base_url): | |
| 145 self.toots = [] | |
| 146 self.media = [] | |
| 147 self.next_mid = 1 | |
| 148 | |
| 149 def log_in(self, username, password, scopes): | |
| 150 return 'TEST_ACCESS_TOKEN' | |
| 151 | |
| 152 def auth_request_url(self, scopes): | |
| 153 return 'https://example.org/auth' | |
| 154 | |
| 155 def status_post(self, toot, media_ids=None, visibility=None): | |
| 156 self.toots.append((toot, media_ids, visibility)) | |
| 157 | |
| 158 def media_post(self, filename, mimetype): | |
| 159 mid = self.next_mid | |
| 160 self.next_mid += 1 | |
| 161 self.media.append((filename, mimetype, mid)) | |
| 162 return mid | |
| 163 | |
| 164 | |
| 165 class MastodonMockUtil: | |
| 166 def installTokens(self, cli, silo_name): | |
| 167 def do_install_tokens(ctx): | |
| 168 ctx.cache.setCustomValue( | |
| 169 '%s_clienttoken' % silo_name, | |
| 170 'TEST_CLIENT_ID,TEST_CLIENT_SECRET') | |
| 171 ctx.cache.setCustomValue( | |
| 172 '%s_accesstoken' % silo_name, | |
| 173 'TEST_ACCESS_TOKEN') | |
| 174 | |
| 175 cli.preExecHook(do_install_tokens) |