Mercurial > silorider
annotate tests/conftest.py @ 53:805c7d768caa
Make feed parser more resilient, add ability to modify html doc.
- Better handle errors found while parsing the html feed.
- Add CSS-selector-based configuration to modify the html feed before parsing
it. This allows using SiloRider with a feed that doesn't have proper
micro-format markup.
| author | Ludovic Chabant <ludovic@chabant.com> |
|---|---|
| date | Sun, 08 Oct 2023 13:52:26 -0700 |
| parents | d3c4c5082bbc |
| children | b404445082f8 |
| rev | line source |
|---|---|
| 0 | 1 import io |
| 2 import os | |
| 3 import os.path | |
| 4 import re | |
| 5 import logging | |
| 6 import tempfile | |
| 7 import pytest | |
| 8 import silorider.main | |
| 9 | |
| 10 | |
| 19 | 11 logger = logging.getLogger(__name__) |
| 12 | |
| 13 | |
| 0 | 14 # def pytest_collect_file(parent, path): |
| 15 # if path.ext == ".html" and path.basename.startswith("feeds"): | |
| 16 # return FeedFile(path, parent) | |
| 17 | |
| 18 | |
| 19 re_feed_test_sep = re.compile(r'^---$') | |
| 20 | |
| 21 | |
| 22 class FeedFile(pytest.File): | |
| 23 def collect(self): | |
| 24 with self.fspath.open() as fp: | |
| 25 markup = fp.read() | |
| 26 | |
| 27 name = self.fspath.basename | |
| 28 html_markup, yaml_markup = re_feed_test_sep.split(markup, 1) | |
| 29 yield FeedItem(name, self, html_markup, yaml_markup) | |
| 30 | |
| 31 | |
| 32 class FeedItem(pytest.Item): | |
| 33 def __init__(self, name, parent, in_spec, out_spec): | |
| 34 super().__init__(name, parent) | |
| 35 self.in_spec = in_spec | |
| 36 self.out_spec = out_spec | |
| 37 | |
| 38 def runtest(self): | |
| 39 pass | |
| 40 | |
| 41 | |
| 42 @pytest.fixture | |
| 43 def cli(): | |
| 44 return CliRunner() | |
| 45 | |
| 46 | |
| 47 class CliRunner: | |
| 48 def __init__(self): | |
| 49 self._cfgtxt = """ | |
| 50 [cache] | |
| 51 uri=memory://for_test | |
| 52 """ | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
53 self._feedcfg = [] |
| 0 | 54 self._pre_hooks = [] |
| 55 self._cleanup = [] | |
| 56 | |
| 57 def getFeedPath(self, name): | |
| 58 return os.path.join(os.path.dirname(__file__), | |
| 59 'feeds', | |
| 60 '%s.html' % name) | |
| 61 | |
| 62 def createTempFeed(self, contents): | |
| 63 tmpfd, tmpname = tempfile.mkstemp() | |
| 64 with os.fdopen(tmpfd, 'w', encoding='utf8') as tmpfp: | |
| 65 tmpfp.write(contents) | |
| 66 self._cleanup.append(tmpname) | |
| 67 return tmpname | |
| 68 | |
| 69 def setConfig(self, cfgtxt): | |
| 70 self._cfgtxt = cfgtxt | |
| 71 return self | |
| 72 | |
| 73 def appendConfig(self, cfgtxt): | |
| 74 self._cfgtxt += cfgtxt | |
| 75 return self | |
| 76 | |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
77 def appendFeedConfig(self, feed_name, feed_url): |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
78 self._feedcfg.append((feed_name, feed_url)) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
79 return self |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
80 |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
81 def setFeedConfig(self, feed_name, feed_url): |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
82 self._feedcfg = [] |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
83 return self.appendFeedConfig(feed_name, feed_url) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
84 |
| 0 | 85 def appendSiloConfig(self, silo_name, silo_type, **options): |
| 86 cfgtxt = '[silo:%s]\n' % silo_name | |
| 87 cfgtxt += 'type=%s\n' % silo_type | |
| 88 if options is not None: | |
| 89 for n, v in options.items(): | |
| 90 cfgtxt += '%s=%s\n' % (n, v) | |
| 91 return self.appendConfig(cfgtxt) | |
| 92 | |
| 93 def preExecHook(self, hook): | |
| 94 self._pre_hooks.append(hook) | |
| 95 | |
| 96 def run(self, *args): | |
| 19 | 97 pre_args = ['-v'] |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
98 if self._cfgtxt or self._feedcfg: |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
99 cfgtxt = self._cfgtxt |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
100 cfgtxt += '\n\n[urls]\n' |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
101 for n, u in self._feedcfg: |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
102 cfgtxt += '%s=%s\n' % (n, u) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
103 |
| 0 | 104 tmpfd, tmpcfg = tempfile.mkstemp() |
| 19 | 105 logger.info("Creating temporary configuration file: %s" % tmpcfg) |
| 0 | 106 with os.fdopen(tmpfd, 'w') as tmpfp: |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
107 tmpfp.write(cfgtxt) |
| 0 | 108 self._cleanup.append(tmpcfg) |
| 19 | 109 pre_args += ['-c', tmpcfg] |
| 0 | 110 |
| 111 captured = io.StringIO() | |
| 112 handler = logging.StreamHandler(captured) | |
| 113 handler.setLevel(logging.INFO) | |
| 114 silorider_logger = logging.getLogger('silorider') | |
| 115 silorider_logger.addHandler(handler) | |
| 116 | |
| 117 main_ctx = None | |
| 118 main_res = None | |
| 119 | |
| 120 def pre_exec_hook(ctx): | |
| 121 for h in self._pre_hooks: | |
| 122 h(ctx) | |
| 123 | |
| 124 def post_exec_hook(ctx, res): | |
| 125 nonlocal main_ctx, main_res | |
| 126 main_ctx = ctx | |
| 127 main_res = res | |
| 128 | |
| 129 silorider.main.pre_exec_hook = pre_exec_hook | |
| 130 silorider.main.post_exec_hook = post_exec_hook | |
| 131 | |
| 132 args = pre_args + list(args) | |
| 19 | 133 logger.info("Running command: %s" % list(args)) |
| 0 | 134 try: |
| 135 silorider.main._unsafe_main(args) | |
| 136 finally: | |
| 137 silorider.main.pre_exec_hook = None | |
| 138 silorider.main.post_exec_hook = None | |
| 139 | |
| 140 silorider_logger.removeHandler(handler) | |
| 141 | |
| 19 | 142 logger.info("Cleaning %d temporary files." % len(self._cleanup)) |
| 0 | 143 for tmpname in self._cleanup: |
|
18
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
144 try: |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
145 os.remove(tmpname) |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
146 except FileNotFoundError: |
|
a921cc2306bc
Do our own HTML parsing/stripping of micropost contents.
Ludovic Chabant <ludovic@chabant.com>
parents:
0
diff
changeset
|
147 pass |
| 0 | 148 |
| 149 return main_ctx, main_res | |
| 150 | |
| 151 | |
| 152 @pytest.fixture | |
| 153 def feedutil(): | |
| 154 return FeedUtil() | |
| 155 | |
| 156 | |
| 157 class FeedUtil: | |
| 158 def makeFeed(self, *entries): | |
| 159 feed = '<html><body>\n' | |
| 160 for e in entries: | |
| 161 feed += '<article class="h-entry">\n' | |
| 162 feed += e | |
| 163 feed += '</article>\n' | |
| 164 feed += '</body></html>' | |
| 165 return feed |