| { |
| "dataset": { |
| "id": "mkultra-foia-mineru-archive", |
| "title": "MKULTRA FOIA MinerU Archive", |
| "version": "2.0.0", |
| "description": "A public-interest release of MinerU-processed MKULTRA and related declassified records comprising an original 16,383-page PEERS FOIA collection and a 4,854-page supplemental collection, with page- and document-level transcriptions, processing metadata, preliminary analysis, source provenance, and experimental redaction mapping.", |
| "collection_count": 2, |
| "page_count": 21237, |
| "primary_record_identifier": "record_id", |
| "legacy_original_page_identifier": "mori_id", |
| "legacy_identifier_example": "0000017354_0009", |
| "language": "en", |
| "publisher": "Public Education and Empowerment Resource Service (PEERS)", |
| "release_scope": "Source-image archives distributed through IPFS, page- and document-level MinerU exports, analysis, provenance metadata, quality annotations, redaction mapping, scripts, and reproducibility notebooks." |
| }, |
| "collections": { |
| "original_foia": { |
| "title": "Original PEERS FOIA Collection", |
| "description": "The original MKULTRA-related records received by PEERS as TIFF files through a Freedom of Information Act request.", |
| "page_count": 16383, |
| "collection_id": "original_foia", |
| "primary_page_identifier": "mori_id", |
| "identifier_example": "0000017354_0009", |
| "source_acquisition": { |
| "method": "Freedom of Information Act request", |
| "recipient": "Public Education and Empowerment Resource Service (PEERS)", |
| "original_format": "TIFF" |
| }, |
| "image_conversion": { |
| "converted_to": "PNG", |
| "year": 2018, |
| "performed_by": "Mark Bailey", |
| "description": "Faithful format conversion of the received TIFF source files." |
| }, |
| "ocr_processing": { |
| "year": 2026, |
| "engine": "MinerU", |
| "mineru_version": "3.4.4", |
| "backend": "pipeline", |
| "torch_version": "2.5.1+cu124", |
| "processing_gpu": "NVIDIA RTX A6000", |
| "gpu_memory_gb": 48 |
| }, |
| "source_images": { |
| "distribution": "IPFS", |
| "archive": { |
| "filename": "ultra_original_pngs.tar.gz", |
| "format": "tar.gz", |
| "size_display": "646.71 MB", |
| "size_bytes": 646582250, |
| "cid": "bafybeicwmhqgjer4il2jkbz255ckn4tq2ianfmy42ehmrpih4ef5yrrdd4", |
| "ipfs_uri": "ipfs://bafybeicwmhqgjer4il2jkbz255ckn4tq2ianfmy42ehmrpih4ef5yrrdd4", |
| "sha256": "5438d3e1807eb6422641fdb107a497c54e150baf715fa2d14e6552f992321f95", |
| "archive_root": "ULTRApng/", |
| "expected_image_count": 16383 |
| }, |
| "checksum_file": { |
| "filename": "ultra_original_pngs.tar.gz.sha256", |
| "repository_path": "checksums/ultra_original_pngs.tar.gz.sha256", |
| "cid": "bafkreia4brfhtgf7pqrtm2twxdfsvvw3jlyatcqyzxm7x2w7w2a7gdkiqq", |
| "ipfs_uri": "ipfs://bafkreia4brfhtgf7pqrtm2twxdfsvvw3jlyatcqyzxm7x2w7w2a7gdkiqq" |
| } |
| }, |
| "principal_files": { |
| "page_level_processed_export": "data/ultra_pages_v2_redactions.jsonl.gz", |
| "document_level_processed_export": "data/ultra_documents_v2_redactions.jsonl.gz", |
| "processing_manifest": "data/ultra_export_manifest_v2_redactions.json", |
| "preliminary_analysis_json": "data/preliminary-analysis.json", |
| "preliminary_analysis_text": "data/preliminary-analysis.txt" |
| } |
| }, |
| "supplemental": { |
| "title": "Supplemental Authenticated Declassified Collection", |
| "description": "Additional MKULTRA and related-program records assembled from additional PEERS FOIA page images and authenticated declassified PDF collections, including materials preserved in Black Vault archival collections.", |
| "page_count": 4854, |
| "collection_id": "supplemental", |
| "primary_page_identifier": "record_id", |
| "source_acquisition": { |
| "method": "Mixed authenticated declassified-document collection", |
| "sources_include": [ |
| "additional PEERS FOIA page images", |
| "declassified MKULTRA and related-program PDFs", |
| "Black Vault archival collections", |
| "other authenticated declassified records documented in the source manifest" |
| ], |
| "provenance_reference": "data/supplemental_source_manifest.csv" |
| }, |
| "image_conversion": { |
| "converted_to": "JPEG", |
| "year": 2026, |
| "performed_by": "Mark Bailey", |
| "maximum_long_side_pixels": 2000, |
| "jpeg_quality": 88, |
| "color_mode": "grayscale", |
| "description": "PDF pages were rendered to images and combined with existing page images. Final MinerU input images were normalized to grayscale JPEG, resized without enlargement to a maximum long side of 2,000 pixels, deduplicated, and screened conservatively for blank pages." |
| }, |
| "ocr_processing": { |
| "year": 2026, |
| "engine": "MinerU", |
| "mineru_version": "3.4.4", |
| "backend": "pipeline", |
| "torch_version": "2.5.1+cu124", |
| "processing_gpu": "NVIDIA RTX A6000", |
| "gpu_memory_gb": 48 |
| }, |
| "source_images": { |
| "distribution": "IPFS", |
| "archive": { |
| "filename": "mkultra_supplemental_images_jpeg.tar.gz", |
| "format": "tar.gz", |
| "cid": "bafybeidpkfhkzj7w2eeemr2dpfbcwj545ygynhh7jjqinihxzyserbu3z4", |
| "ipfs_uri": "ipfs://bafybeidpkfhkzj7w2eeemr2dpfbcwj545ygynhh7jjqinihxzyserbu3z4", |
| "sha256": "000756859d003e001032cf9583c86b873c8c9b5ec2e7620c0532ebbdbf028891", |
| "archive_root": "pages/", |
| "expected_image_count": 4854 |
| }, |
| "checksum_file": { |
| "filename": "mkultra_supplemental_images_jpeg.tar.gz.sha256", |
| "repository_path": "checksums/mkultra_supplemental_images_jpeg.tar.gz.sha256", |
| "cid": "bafkreigxuzgbnaorxa6pqhntnkymq2i7om6x7bu5tbcvfiny6gfewcgo54", |
| "ipfs_uri": "ipfs://bafkreigxuzgbnaorxa6pqhntnkymq2i7om6x7bu5tbcvfiny6gfewcgo54" |
| } |
| }, |
| "principal_files": { |
| "page_level_processed_export": "data/supplemental_pages_v1_redactions.jsonl.gz", |
| "document_level_processed_export": "data/supplemental_documents_v1_redactions.jsonl.gz", |
| "processing_manifest": "data/supplemental_export_manifest_v1.json", |
| "preliminary_analysis_json": "data/supplemental_preliminary_analysis.json", |
| "preliminary_analysis_text": "data/supplemental_preliminary_analysis.txt", |
| "source_inventory": "data/supplemental_source_inventory.csv", |
| "source_manifest": "data/supplemental_source_manifest.csv", |
| "missing_output_report": "data/supplemental_missing_mineru_outputs.csv", |
| "orphan_output_report": "data/supplemental_orphan_mineru_outputs.csv" |
| } |
| } |
| }, |
| "rights": { |
| "source_documents": { |
| "status": "Treated as public-domain United States federal government records, subject to the possibility that individual pages may contain third-party material.", |
| "exclusive_rights_asserted": false |
| }, |
| "image_conversions": { |
| "exclusive_rights_asserted": false, |
| "note": "The PNG and JPEG files are faithful technical conversions or normalized derivatives of the source government records." |
| }, |
| "original_processing_contributions": { |
| "license": "MIT", |
| "license_file": "LICENSE", |
| "applies_to": [ |
| "processing scripts", |
| "analysis scripts", |
| "notebooks", |
| "original metadata organization", |
| "source provenance organization", |
| "quality annotations", |
| "experimental redaction annotations", |
| "other original contributions to the extent protectable" |
| ] |
| }, |
| "commercial_use": true, |
| "attribution_requested": true, |
| "attribution_required_where_mit_applies": true |
| }, |
| "redaction_mapping": { |
| "status": "experimental", |
| "applies_to": [ |
| "original_foia", |
| "supplemental" |
| ], |
| "description": "Probable dark redaction regions detected heuristically from source-page images.", |
| "annotations_may_include": [ |
| "generated marker such as (redacted_12)", |
| "estimated obscured character-space count", |
| "lower and upper estimate bounds", |
| "pixel-space bounding box", |
| "fill ratio", |
| "aspect ratio", |
| "character-pitch estimate", |
| "character-pitch estimation method", |
| "confidence score" |
| ], |
| "warning": "These annotations are heuristic and do not reconstruct concealed text. Dark graphical elements, stamps, borders, photocopy artifacts, and other page features may produce false positives." |
| }, |
| "compatibility": { |
| "original_exports_preserved": true, |
| "original_record_identifiers_preserved": true, |
| "supplemental_records_namespaced": true, |
| "notes": "The original v2 exports remain unchanged. Supplemental records are distributed as separate page- and document-level exports so existing users are not affected." |
| }, |
| "excluded_convenience_artifacts": [ |
| "cleaned production search index", |
| "browser-optimized page index", |
| "thumbnail collection", |
| "search ranking implementation", |
| "static browser application", |
| "deployment package" |
| ] |
| } |
