DL fix again

This commit is contained in:
Dobromir Popov
2026-07-06 19:27:37 +03:00
parent b615acf582
commit 4f007aeef9
4 changed files with 142 additions and 20 deletions

View File

@@ -37,7 +37,8 @@ def compute_shard_checksum(shard_dir: Path) -> str:
def write_shard_archive(shard_dir: Path, out_file: Any) -> None:
"""Write a tar archive for *shard_dir* to a binary file-like object."""
with tarfile.open(fileobj=out_file, mode="w|") as archive:
# dereference: HF cache snapshots are symlinks into blobs/ — ship contents.
with tarfile.open(fileobj=out_file, mode="w|", dereference=True) as archive:
for path in sorted(p for p in shard_dir.rglob("*") if p.is_file()):
archive.add(path, arcname=path.relative_to(shard_dir).as_posix())