Skip to content

Commit 1cf59ea

Browse files
XiuShenAlXiuShenAl
authored
refactor(plugins): retain old plugin versions on release and clean up failed load state (#5695)
1 parent e2f9464 commit 1cf59ea

5 files changed

Lines changed: 723 additions & 75 deletions

File tree

‎.github/workflows/plugins-release.yml‎

Lines changed: 16 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ jobs:
5757
5858
- name: Sync plugin zips to OSS (long-cache, immutable)
5959
run: |
60+
# nullglob ensures empty globs expand to nothing in this bash session
6061
shopt -s nullglob
6162
for kind in bundle tool; do
6263
while IFS= read -r -d '' f; do
@@ -69,10 +70,9 @@ jobs:
6970
--meta "Cache-Control:public, max-age=31536000, immutable"
7071
done < <(find "dist/plugins/${kind}" -type f -name '*.zip' -print0 2>/dev/null || true)
7172
done
72-
# Remove zips that no longer exist locally so deletions propagate.
73+
# One-time cleanup: remove legacy flat zips (pre per-plugin directory layout).
74+
# This only targets {kind}/*.zip (flat files), not {kind}/{plugin_id}/*.zip (versioned).
7375
for kind in bundle tool; do
74-
kind_root="dist/plugins/${kind}"
75-
# Drop legacy flat zips at {kind}/ (pre per-plugin directory layout).
7676
remote_flat=$(ossutil ls "oss://qwenpaw-download/files/plugins/${kind}/" -s 2>/dev/null \
7777
| grep -E "^oss://qwenpaw-download/files/plugins/${kind}/[^/]+\.zip$" \
7878
| sed 's|.*/||' \
@@ -83,25 +83,18 @@ jobs:
8383
"oss://qwenpaw-download/files/plugins/${kind}/${name}" \
8484
--force
8585
done
86-
for plugin_dir in "$kind_root"/*/; do
87-
[ -d "$plugin_dir" ] || continue
88-
plugin_id=$(basename "$plugin_dir")
89-
local_names=$(find "$plugin_dir" -maxdepth 1 -type f -name '*.zip' -exec basename {} \; 2>/dev/null | sort || true)
90-
remote_names=$(ossutil ls "oss://qwenpaw-download/files/plugins/${kind}/${plugin_id}/" -s 2>/dev/null \
91-
| grep -E "^oss://qwenpaw-download/files/plugins/${kind}/${plugin_id}/[^/]+\.zip$" \
92-
| sed 's|.*/||' \
93-
| sort -u || true)
94-
for name in $remote_names; do
95-
if ! echo "$local_names" | grep -qx "$name"; then
96-
echo "Removing stale OSS object: ${kind}/${plugin_id}/${name}"
97-
ossutil rm \
98-
"oss://qwenpaw-download/files/plugins/${kind}/${plugin_id}/${name}" \
99-
--force
100-
fi
101-
done
102-
done
10386
done
10487
88+
- name: Merge historical versions into index
89+
run: |
90+
ossutil cp "oss://qwenpaw-download/metadata/plugins/index.json" \
91+
existing-index.json 2>/dev/null || echo '{}' > existing-index.json
92+
93+
python3 scripts/pack/merge_plugin_index.py \
94+
--new dist/plugins/index.json \
95+
--old existing-index.json \
96+
--out dist/plugins/index.json
97+
10598
- name: Upload plugins index (short-cache)
10699
run: |
107100
ossutil cp dist/plugins/index.json \
@@ -121,23 +114,9 @@ jobs:
121114
}
122115
EOF
123116
124-
python3 << 'PYTHON'
125-
import json
126-
from datetime import datetime, timezone
127-
128-
with open("main-index.json", "r", encoding="utf-8") as f:
129-
index = json.load(f)
130-
131-
index.setdefault("products", {})
132-
index["products"]["plugins"] = {
133-
"name": {"zh-CN": "插件", "en-US": "Plugins"},
134-
"index_url": "/metadata/plugins/index.json",
135-
}
136-
index["updated_at"] = datetime.now(timezone.utc).isoformat()
137-
138-
with open("main-index.json", "w", encoding="utf-8") as f:
139-
json.dump(index, f, indent=2, ensure_ascii=False)
140-
PYTHON
117+
python3 scripts/pack/patch_main_index.py \
118+
--index main-index.json \
119+
--out main-index.json
141120
142121
ossutil cp main-index.json \
143122
"oss://qwenpaw-download/metadata/index.json" \

‎scripts/pack/merge_plugin_index.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
#!/usr/bin/env python3
2+
"""Merge a newly built plugin index with a historical one from OSS.
3+
4+
Used by the ``plugins-release.yml`` workflow to preserve old plugin
5+
versions on the CDN while adding new ones.
6+
7+
Merge rules:
8+
- ``files``: keyed by file_id (``{plugin_id}-{version}``). New entries
9+
overwrite same-id old entries; different ids from old are preserved.
10+
- ``platforms.{kind}.versions``: union of new and old version lists,
11+
new versions first, old versions appended with deduplication.
12+
13+
Usage::
14+
15+
python scripts/pack/merge_plugin_index.py \
16+
--new dist/plugins/index.json \
17+
--old existing-index.json \
18+
--out dist/plugins/index.json
19+
"""
20+
21+
from __future__ import annotations
22+
23+
import argparse
24+
import json
25+
from pathlib import Path
26+
27+
28+
def merge_indexes(new_index: dict, old_index: dict) -> dict:
29+
"""Merge *old_index* into *new_index* (mutates and returns *new_index*)."""
30+
# files: same file_id is overwritten by new; different ids preserved.
31+
old_files = old_index.get("files", {})
32+
new_files = new_index.get("files", {})
33+
new_index["files"] = {**old_files, **new_files}
34+
35+
# platforms.versions: union, new first, old appended with dedup.
36+
old_platforms = old_index.get("platforms", {})
37+
all_kinds = set(
38+
list(new_index.get("platforms", {}).keys())
39+
+ list(old_platforms.keys()),
40+
)
41+
for kind in all_kinds:
42+
old_versions = old_platforms.get(kind, {}).get("versions", [])
43+
new_versions = (
44+
new_index.get("platforms", {}).get(kind, {}).get("versions", [])
45+
)
46+
seen = set(new_versions)
47+
merged = list(new_versions)
48+
for v in old_versions:
49+
if v not in seen:
50+
merged.append(v)
51+
seen.add(v)
52+
new_index.setdefault("platforms", {})
53+
new_index["platforms"].setdefault(kind, {})
54+
new_index["platforms"][kind]["versions"] = merged
55+
56+
return new_index
57+
58+
59+
def main(argv: list[str] | None = None) -> None:
60+
parser = argparse.ArgumentParser(
61+
description="Merge new and historical plugin indexes.",
62+
)
63+
parser.add_argument(
64+
"--new",
65+
required=True,
66+
type=Path,
67+
help="Path to the newly generated index.json",
68+
)
69+
parser.add_argument(
70+
"--old",
71+
required=True,
72+
type=Path,
73+
help="Path to the existing (historical) index.json",
74+
)
75+
parser.add_argument(
76+
"--out",
77+
required=True,
78+
type=Path,
79+
help="Output path for the merged index.json",
80+
)
81+
args = parser.parse_args(argv)
82+
83+
with open(args.new, encoding="utf-8") as f:
84+
new_index = json.load(f)
85+
with open(args.old, encoding="utf-8") as f:
86+
old_index = json.load(f)
87+
88+
merged = merge_indexes(new_index, old_index)
89+
90+
with open(args.out, "w", encoding="utf-8") as f:
91+
json.dump(merged, f, indent=2, ensure_ascii=False)
92+
93+
94+
if __name__ == "__main__":
95+
main()

‎scripts/pack/patch_main_index.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env python3
2+
"""Patch the main OSS metadata index to advertise the plugins product.
3+
4+
Ensures the top-level ``metadata/index.json`` has a ``products.plugins``
5+
entry pointing to the plugins sub-index.
6+
7+
Usage::
8+
9+
python scripts/pack/patch_main_index.py \
10+
--index main-index.json \
11+
--out main-index.json
12+
"""
13+
14+
from __future__ import annotations
15+
16+
import argparse
17+
import json
18+
from datetime import datetime, timezone
19+
from pathlib import Path
20+
21+
22+
def patch_index(index: dict) -> dict:
23+
"""Add/update the ``plugins`` product entry (mutates *index*)."""
24+
index.setdefault("products", {})
25+
index["products"]["plugins"] = {
26+
"name": {"zh-CN": "插件", "en-US": "Plugins"},
27+
"index_url": "/metadata/plugins/index.json",
28+
}
29+
index["updated_at"] = datetime.now(timezone.utc).isoformat()
30+
return index
31+
32+
33+
def main(argv: list[str] | None = None) -> None:
34+
parser = argparse.ArgumentParser(
35+
description="Patch main metadata index with plugins product entry.",
36+
)
37+
parser.add_argument(
38+
"--index",
39+
required=True,
40+
type=Path,
41+
help="Path to the main index.json",
42+
)
43+
parser.add_argument(
44+
"--out",
45+
required=True,
46+
type=Path,
47+
help="Output path for the patched index.json",
48+
)
49+
args = parser.parse_args(argv)
50+
51+
with open(args.index, encoding="utf-8") as f:
52+
index = json.load(f)
53+
54+
patched = patch_index(index)
55+
56+
with open(args.out, "w", encoding="utf-8") as f:
57+
json.dump(patched, f, indent=2, ensure_ascii=False)
58+
59+
60+
if __name__ == "__main__":
61+
main()

‎src/qwenpaw/plugins/loader.py‎

Lines changed: 102 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -404,49 +404,113 @@ async def _load_backend_module(
404404
)
405405

406406
module = importlib.util.module_from_spec(spec)
407-
sys.modules[module_name] = module
408-
module.__package__ = module_name
409-
module.__path__ = [plugin_dir_str]
410-
spec.loader.exec_module(module)
411-
412-
if not hasattr(module, "plugin"):
413-
raise AttributeError(
414-
"Plugin module must export 'plugin' object",
415-
)
416407

417-
plugin_def = module.plugin
418-
419-
manifest_dict = {
420-
"id": manifest.id,
421-
"name": manifest.name,
422-
"version": manifest.version,
423-
"description": manifest.description,
424-
"author": manifest.author,
425-
"dependencies": manifest.dependencies,
426-
"min_version": manifest.min_version,
427-
"max_version": manifest.max_version,
428-
"qwenpaw_version": (
429-
manifest.qwenpaw_version.model_dump()
430-
if manifest.qwenpaw_version
431-
else None
432-
),
433-
"meta": manifest.meta,
434-
}
435-
api = PluginApi(plugin_id, config or {}, manifest_dict)
436-
api.set_registry(self.registry)
437-
self.registry.register_plugin_manifest(plugin_id, manifest_dict)
438-
439-
if hasattr(plugin_def, "register"):
440-
result = plugin_def.register(api)
441-
if inspect.iscoroutine(result) or inspect.isawaitable(result):
442-
await result
443-
else:
444-
raise AttributeError(
445-
"Plugin must implement 'register(api)' method",
408+
try:
409+
sys.modules[module_name] = module
410+
module.__package__ = module_name
411+
module.__path__ = [plugin_dir_str]
412+
spec.loader.exec_module(module)
413+
414+
if not hasattr(module, "plugin"):
415+
raise AttributeError(
416+
"Plugin module must export 'plugin' object",
417+
)
418+
419+
plugin_def = module.plugin
420+
421+
if manifest.qwenpaw_version is not None:
422+
qv_dict = manifest.qwenpaw_version.model_dump()
423+
else:
424+
qv_dict = {
425+
"min": manifest.min_version,
426+
"max": manifest.max_version,
427+
}
428+
manifest_dict = {
429+
"id": manifest.id,
430+
"name": manifest.name,
431+
"version": manifest.version,
432+
"description": manifest.description,
433+
"author": manifest.author,
434+
"dependencies": manifest.dependencies,
435+
"qwenpaw_version": qv_dict,
436+
"meta": manifest.meta,
437+
}
438+
api = PluginApi(plugin_id, config or {}, manifest_dict)
439+
api.set_registry(self.registry)
440+
self.registry.register_plugin_manifest(plugin_id, manifest_dict)
441+
442+
if hasattr(plugin_def, "register"):
443+
result = plugin_def.register(api)
444+
if inspect.iscoroutine(result) or inspect.isawaitable(result):
445+
await result
446+
else:
447+
raise AttributeError(
448+
"Plugin must implement 'register(api)' method",
449+
)
450+
except Exception:
451+
self._cleanup_failed_load(
452+
plugin_id,
453+
module_name,
454+
source_path,
446455
)
456+
raise
447457

448458
return plugin_def
449459

460+
def _cleanup_failed_load(
461+
self,
462+
plugin_id: str,
463+
module_name: str,
464+
source_path: Path,
465+
) -> None:
466+
"""Roll back side effects after a failed plugin load.
467+
468+
Mirrors the cleanup logic in ``unload_plugin`` (registry,
469+
``sys.modules``, ``sys.path``) so that a failed load leaves no
470+
orphan state that could interfere with other plugins or a
471+
subsequent retry.
472+
473+
.. note::
474+
NOT thread-safe. ``sys.modules`` and ``sys.path`` mutations
475+
are not guarded by a lock. This is fine because
476+
``load_all_plugins`` loads plugins sequentially, but callers
477+
must not invoke this method concurrently.
478+
"""
479+
logger.warning(
480+
"Cleaning up failed plugin load for '%s'",
481+
plugin_id,
482+
)
483+
484+
# 1. Registry (manifest, providers, hooks, middleware, routes, …)
485+
self.registry.unregister_plugin(plugin_id)
486+
487+
# 2. sys.modules — by module-name prefix
488+
prefix = module_name + "."
489+
stale = [
490+
k for k in sys.modules if k == module_name or k.startswith(prefix)
491+
]
492+
for k in stale:
493+
sys.modules.pop(k, None)
494+
495+
# 3. sys.modules — by __file__ path (catches bare imports that
496+
# bypassed the plugin_<id> namespace, e.g. ``import utils``
497+
# after the plugin inserted its dir into sys.path).
498+
source_resolved = os.path.realpath(str(source_path)) + os.sep
499+
stale_by_file = [
500+
k
501+
for k, mod in list(sys.modules.items())
502+
if (mod_file := getattr(mod, "__file__", None)) is not None
503+
and os.path.realpath(mod_file).startswith(source_resolved)
504+
]
505+
for k in stale_by_file:
506+
sys.modules.pop(k, None)
507+
508+
# 4. sys.path — remove the plugin directory if it was added
509+
plugin_dir_real = os.path.realpath(str(source_path))
510+
sys.path[:] = [
511+
p for p in sys.path if os.path.realpath(p) != plugin_dir_real
512+
]
513+
450514
async def load_plugin(
451515
self,
452516
manifest: PluginManifest,

0 commit comments

Comments
 (0)