282 lines
12 KiB
Python
282 lines
12 KiB
Python
import importlib.util
|
||
import shutil
|
||
import tempfile
|
||
import threading
|
||
import unittest
|
||
from datetime import date
|
||
from http.server import SimpleHTTPRequestHandler
|
||
from pathlib import Path
|
||
from socketserver import TCPServer
|
||
|
||
|
||
SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "clip_web_article.py"
|
||
spec = importlib.util.spec_from_file_location("clip_web_article", SCRIPT_PATH)
|
||
clip = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(clip)
|
||
|
||
|
||
class ClipWebArticleTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = Path(tempfile.mkdtemp(prefix="clip-web-article-test-", dir=Path.cwd() / "tmp"))
|
||
|
||
def tearDown(self):
|
||
shutil.rmtree(self.tmp)
|
||
|
||
def test_normalize_opencli_article_flattens_images_and_removes_added_metadata(self):
|
||
source = self.tmp / "opencli"
|
||
article_dir = source / "BadTitle"
|
||
images_dir = article_dir / "images"
|
||
images_dir.mkdir(parents=True)
|
||
(images_dir / "img_001.jpg").write_bytes(b"image-one")
|
||
(article_dir / "BadTitle.md").write_text(
|
||
"\n".join(
|
||
[
|
||
"# Bad: Title?/ Example",
|
||
"",
|
||
"> 作者: Alice",
|
||
"> 发布时间: 2026-06-26 10:00:00",
|
||
"> 原文链接: https://example.test/a",
|
||
"",
|
||
"---",
|
||
"",
|
||
"First paragraph.",
|
||
"",
|
||
"",
|
||
"",
|
||
"",
|
||
"",
|
||
]
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
result = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27))
|
||
|
||
self.assertEqual(result.image_count, 1)
|
||
self.assertTrue(result.output_dir.name.startswith("20260626-"))
|
||
self.assertNotRegex(result.output_dir.name, r'[<>:"/\\|?*]')
|
||
markdown_files = list(result.output_dir.glob("*.md"))
|
||
self.assertEqual(len(markdown_files), 1)
|
||
md_text = markdown_files[0].read_text(encoding="utf-8")
|
||
self.assertTrue(md_text.startswith("# Bad: Title?/ Example\n\n"))
|
||
self.assertNotIn("原文链接", md_text)
|
||
self.assertNotIn("> 作者:", md_text)
|
||
self.assertNotIn("> 发布时间:", md_text)
|
||
self.assertNotIn("\n---\n", md_text)
|
||
self.assertIn("", md_text)
|
||
self.assertIn("", md_text)
|
||
self.assertEqual((result.output_dir / "img_1.jpg").read_bytes(), b"image-one")
|
||
|
||
def test_normalize_opencli_article_never_overwrites_existing_folder(self):
|
||
source = self.tmp / "opencli"
|
||
article_dir = source / "Title"
|
||
article_dir.mkdir(parents=True)
|
||
(article_dir / "Title.md").write_text("# Same Title\n\nBody\n", encoding="utf-8")
|
||
|
||
first = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27))
|
||
second = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27))
|
||
|
||
self.assertNotEqual(first.output_dir, second.output_dir)
|
||
self.assertTrue(second.output_dir.name.endswith("-2"))
|
||
self.assertTrue((first.output_dir / "Same Title.md").exists())
|
||
self.assertTrue((second.output_dir / "Same Title.md").exists())
|
||
|
||
def test_save_twitter_rows_keeps_only_target_status_and_downloads_images(self):
|
||
image_dir = self.tmp / "server"
|
||
image_dir.mkdir()
|
||
(image_dir / "photo.jpg").write_bytes(b"tweet-image")
|
||
|
||
class QuietHandler(SimpleHTTPRequestHandler):
|
||
def log_message(self, format, *args):
|
||
return
|
||
|
||
old_cwd = Path.cwd()
|
||
server = TCPServer(("127.0.0.1", 0), QuietHandler)
|
||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||
try:
|
||
import os
|
||
|
||
os.chdir(image_dir)
|
||
thread.start()
|
||
image_url = f"http://127.0.0.1:{server.server_address[1]}/photo.jpg"
|
||
rows = [
|
||
{
|
||
"id": "1",
|
||
"author": "alice",
|
||
"text": "AI agents should preserve original English text.",
|
||
"created_at": "Wed Jun 24 04:02:06 +0000 2026",
|
||
"media_urls": [image_url],
|
||
},
|
||
{
|
||
"id": "2",
|
||
"author": "bob",
|
||
"text": "@alice This reply should not be saved.",
|
||
"in_reply_to": "1",
|
||
"media_urls": [image_url],
|
||
},
|
||
]
|
||
|
||
result = clip.save_twitter_rows(
|
||
rows,
|
||
self.tmp / "out",
|
||
"https://x.com/alice/status/1",
|
||
date(2026, 6, 27),
|
||
)
|
||
finally:
|
||
server.shutdown()
|
||
server.server_close()
|
||
import os
|
||
|
||
os.chdir(old_cwd)
|
||
|
||
md_text = result.markdown_path.read_text(encoding="utf-8")
|
||
self.assertIn("AI agents should preserve original English text.", md_text)
|
||
self.assertNotIn("This reply should not be saved", md_text)
|
||
self.assertNotIn("**@alice**", md_text)
|
||
self.assertNotIn("**@bob**", md_text)
|
||
self.assertIn("", md_text)
|
||
self.assertNotIn("https://x.com/alice/status/1", md_text)
|
||
self.assertEqual((result.output_dir / "img_1.jpg").read_bytes(), b"tweet-image")
|
||
self.assertEqual(result.image_count, 1)
|
||
self.assertEqual(result.language, "english")
|
||
self.assertTrue(result.output_dir.name.startswith("20260624-"))
|
||
self.assertLessEqual(len(result.output_dir.name), 30)
|
||
self.assertLessEqual(len(result.markdown_path.stem), 32)
|
||
|
||
def test_normalize_zhihu_pin_web_read_keeps_article_body_and_drops_page_chrome(self):
|
||
source = self.tmp / "opencli"
|
||
article_dir = source / "Pin"
|
||
images_dir = article_dir / "images"
|
||
images_dir.mkdir(parents=True)
|
||
(images_dir / "img_003.jpg").write_bytes(b"cover")
|
||
(article_dir / "Pin.md").write_text(
|
||
"\n".join(
|
||
[
|
||
"# OpenAI内部是怎么使用Codex？ | 最近OpenAI公布了其内部是如何使用Codex。为了方便大家阅读，我基于OpenA…",
|
||
"> 原文链接: https://www.zhihu.com/pin/2042210955053065858",
|
||
"",
|
||
"---",
|
||
"",
|
||
"[](https://www.zhihu.com/people/yang-chao-62)",
|
||
"AI生命克劳德",
|
||
"65 人赞同了该想法",
|
||
"",
|
||
"OpenAI内部是怎么使用Codex？ |",
|
||
"",
|
||
"最近OpenAI公布了其内部是如何使用Codex。",
|
||
"",
|
||
"",
|
||
"",
|
||
"+5",
|
||
"",
|
||
"[编辑于2026-05-25 20:36](https://www.zhihu.com/pin/2042210955053065858)・IP 属地上海",
|
||
"2 条评论",
|
||
"默认",
|
||
"最新评论不应该保存",
|
||
]
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
result = clip.normalize_opencli_article(source, self.tmp / "out", date(2026, 6, 27), source_kind="zhihu-pin")
|
||
|
||
md_text = result.markdown_path.read_text(encoding="utf-8")
|
||
self.assertTrue(result.output_dir.name.startswith("20260525-"))
|
||
self.assertIn("最近OpenAI公布了其内部是如何使用Codex。", md_text)
|
||
self.assertIn("(img_1.jpg)", md_text)
|
||
self.assertNotIn("AI生命克劳德", md_text)
|
||
self.assertNotIn("最新评论不应该保存", md_text)
|
||
self.assertNotIn("原文链接", md_text)
|
||
self.assertLessEqual(len(result.output_dir.name), 30)
|
||
self.assertLessEqual(len(result.markdown_path.stem), 32)
|
||
|
||
def test_route_prefers_opencli_without_using_cookie_files(self):
|
||
zhihu_answer = clip.plan_opencli_invocation(
|
||
"https://www.zhihu.com/question/1998112125915267964/answer/2052741315209831340",
|
||
self.tmp / "work",
|
||
)
|
||
zhihu_pin = clip.plan_opencli_invocation("https://www.zhihu.com/pin/2042210955053065858", self.tmp / "work")
|
||
weixin = clip.plan_opencli_invocation("https://mp.weixin.qq.com/s/TApHplXcBONvmVXgDKUUOg", self.tmp / "work")
|
||
x_url = clip.plan_opencli_invocation("https://x.com/dotey/status/2069632132431929651", self.tmp / "work")
|
||
|
||
self.assertEqual(zhihu_answer.kind, "opencli-zhihu-answer-detail")
|
||
self.assertIn("zhihu", zhihu_answer.command)
|
||
self.assertIn("answer-detail", zhihu_answer.command)
|
||
self.assertIn("2052741315209831340", zhihu_answer.command)
|
||
self.assertEqual(zhihu_pin.kind, "opencli-zhihu-pin-web-read")
|
||
self.assertEqual(weixin.kind, "opencli-download")
|
||
self.assertIn("weixin", weixin.command)
|
||
self.assertEqual(x_url.kind, "opencli-twitter-thread")
|
||
self.assertIn("twitter", x_url.command)
|
||
joined = " ".join(zhihu_answer.command + zhihu_pin.command + weixin.command + x_url.command)
|
||
self.assertNotIn("cookie", joined.lower())
|
||
|
||
def test_opencli_plans_release_browser_tabs(self):
|
||
urls = [
|
||
"https://www.zhihu.com/question/1998112125915267964/answer/2052741315209831340",
|
||
"https://www.zhihu.com/pin/2042210955053065858",
|
||
"https://mp.weixin.qq.com/s/TApHplXcBONvmVXgDKUUOg",
|
||
"https://x.com/dotey/status/2069632132431929651",
|
||
"https://example.com/article",
|
||
]
|
||
|
||
for url in urls:
|
||
with self.subTest(url=url):
|
||
command = clip.plan_opencli_invocation(url, self.tmp / "work").command
|
||
self.assertIn("--keep-tab", command)
|
||
keep_tab_index = command.index("--keep-tab")
|
||
self.assertEqual(command[keep_tab_index + 1], "false")
|
||
|
||
def test_parser_accepts_batch_urls_and_custom_output_root(self):
|
||
args = clip.build_parser().parse_args(
|
||
[
|
||
"https://example.test/one",
|
||
"https://example.test/two",
|
||
"--output-root",
|
||
str(self.tmp / "custom-out"),
|
||
]
|
||
)
|
||
|
||
self.assertEqual(args.urls, ["https://example.test/one", "https://example.test/two"])
|
||
self.assertEqual(Path(args.output_root), self.tmp / "custom-out")
|
||
|
||
def test_clip_urls_processes_batch_with_distinct_temporary_work_dirs(self):
|
||
calls = []
|
||
old_clip_url = clip.clip_url
|
||
try:
|
||
def fake_clip_url(url, output_root, work_dir, capture_date=None):
|
||
calls.append((url, output_root, work_dir))
|
||
work_dir.mkdir(parents=True, exist_ok=True)
|
||
md_path = output_root / f"{url[-3:]}.md"
|
||
return clip.ClipResult(url, output_root / url[-3:], md_path, 0, "english")
|
||
|
||
clip.clip_url = fake_clip_url
|
||
items = clip.clip_urls(
|
||
["https://example.test/one", "https://example.test/two"],
|
||
self.tmp / "out",
|
||
self.tmp / "work",
|
||
keep_work=False,
|
||
capture_date=date(2026, 6, 27),
|
||
)
|
||
finally:
|
||
clip.clip_url = old_clip_url
|
||
|
||
self.assertEqual([item.url for item in items], ["https://example.test/one", "https://example.test/two"])
|
||
self.assertTrue(all(item.result is not None and not item.error for item in items))
|
||
self.assertEqual(len({work_dir for _url, _output_root, work_dir in calls}), 2)
|
||
self.assertFalse(any(work_dir.exists() for _url, _output_root, work_dir in calls))
|
||
|
||
def test_format_opencli_failure_explains_browser_bridge_exit_code(self):
|
||
reason = clip.format_opencli_failure(
|
||
69,
|
||
"Error: Browser Bridge extension not connected\n exitCode: 69\n",
|
||
"",
|
||
)
|
||
|
||
self.assertIn("Browser Bridge", reason)
|
||
self.assertIn("not connected", reason)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|