chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Pre-flight environment validation for nf-core pipelines.
|
||||
|
||||
Checks Docker, Nextflow, Java, system resources, and network connectivity.
|
||||
Run this BEFORE attempting any pipeline execution.
|
||||
|
||||
Usage:
|
||||
python check_environment.py
|
||||
python check_environment.py --json
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckResult:
|
||||
"""Result of a single environment check."""
|
||||
name: str
|
||||
passed: bool
|
||||
message: str
|
||||
details: Optional[str] = None
|
||||
fix: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class EnvironmentReport:
|
||||
"""Complete environment validation report."""
|
||||
ready: bool
|
||||
checks: List[CheckResult] = field(default_factory=list)
|
||||
recommendations: List[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"ready": self.ready,
|
||||
"checks": [asdict(c) for c in self.checks],
|
||||
"recommendations": self.recommendations
|
||||
}
|
||||
|
||||
|
||||
def check_docker() -> CheckResult:
|
||||
"""Check Docker availability, daemon status, and permissions."""
|
||||
if not shutil.which("docker"):
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message="Docker not found in PATH",
|
||||
fix="Install Docker: https://docs.docker.com/get-docker/"
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "info"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
stderr_lower = result.stderr.lower()
|
||||
if "permission denied" in stderr_lower:
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message="Docker permission denied",
|
||||
details="Cannot connect to Docker daemon",
|
||||
fix="sudo usermod -aG docker $USER && newgrp docker"
|
||||
)
|
||||
elif "cannot connect" in stderr_lower or "is the docker daemon running" in stderr_lower:
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message="Docker daemon not running",
|
||||
details=result.stderr[:200] if result.stderr else None,
|
||||
fix="sudo systemctl start docker"
|
||||
)
|
||||
else:
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message="Docker error",
|
||||
details=result.stderr[:200] if result.stderr else None,
|
||||
fix="Check Docker installation and daemon status"
|
||||
)
|
||||
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=True,
|
||||
message="Docker is available and running"
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message="Docker command timed out",
|
||||
fix="Check Docker daemon status: sudo systemctl status docker"
|
||||
)
|
||||
except Exception as e:
|
||||
return CheckResult(
|
||||
name="Docker",
|
||||
passed=False,
|
||||
message=f"Docker check failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def check_nextflow() -> CheckResult:
|
||||
"""Check Nextflow installation and version (requires >= 23.04)."""
|
||||
if not shutil.which("nextflow"):
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=False,
|
||||
message="Nextflow not found in PATH",
|
||||
fix="curl -s https://get.nextflow.io | bash && mv nextflow ~/bin/ && export PATH=$HOME/bin:$PATH"
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["nextflow", "-version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
output = result.stdout + result.stderr
|
||||
version_line = output.strip().split('\n')[0] if output else ""
|
||||
|
||||
import re
|
||||
match = re.search(r'(\d+)\.(\d+)\.(\d+)', version_line)
|
||||
|
||||
if match:
|
||||
major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
version_str = f"{major}.{minor}.{patch}"
|
||||
|
||||
# Require version >= 23.04
|
||||
if major > 23 or (major == 23 and minor >= 4):
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=True,
|
||||
message=f"Nextflow {version_str} installed",
|
||||
details=version_line
|
||||
)
|
||||
else:
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=False,
|
||||
message=f"Nextflow {version_str} is outdated (requires >= 23.04)",
|
||||
details=version_line,
|
||||
fix="nextflow self-update"
|
||||
)
|
||||
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=True,
|
||||
message="Nextflow installed (version unknown)",
|
||||
details=version_line
|
||||
)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=False,
|
||||
message="Nextflow command timed out",
|
||||
fix="Check Nextflow installation"
|
||||
)
|
||||
except Exception as e:
|
||||
return CheckResult(
|
||||
name="Nextflow",
|
||||
passed=False,
|
||||
message=f"Nextflow check failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def check_java() -> CheckResult:
|
||||
"""Check Java version (requires >= 11)."""
|
||||
if not shutil.which("java"):
|
||||
return CheckResult(
|
||||
name="Java",
|
||||
passed=False,
|
||||
message="Java not found in PATH",
|
||||
fix="Install Java 11+: sudo apt install openjdk-11-jdk"
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["java", "-version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10
|
||||
)
|
||||
|
||||
# Java version is typically in stderr
|
||||
output = result.stderr or result.stdout
|
||||
import re
|
||||
match = re.search(r'version "(\d+)', output)
|
||||
|
||||
if match:
|
||||
version = int(match.group(1))
|
||||
version_line = output.strip().split('\n')[0]
|
||||
|
||||
if version >= 11:
|
||||
return CheckResult(
|
||||
name="Java",
|
||||
passed=True,
|
||||
message=f"Java {version} installed",
|
||||
details=version_line
|
||||
)
|
||||
else:
|
||||
return CheckResult(
|
||||
name="Java",
|
||||
passed=False,
|
||||
message=f"Java {version} is too old (requires >= 11)",
|
||||
details=version_line,
|
||||
fix="Install Java 11+: sudo apt install openjdk-11-jdk"
|
||||
)
|
||||
|
||||
return CheckResult(
|
||||
name="Java",
|
||||
passed=True,
|
||||
message="Java installed",
|
||||
details=output.strip().split('\n')[0] if output else None
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return CheckResult(
|
||||
name="Java",
|
||||
passed=False,
|
||||
message=f"Java check failed: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def check_resources() -> CheckResult:
|
||||
"""Check system resources (CPU, memory, disk)."""
|
||||
try:
|
||||
# CPU cores
|
||||
cpu_count = os.cpu_count() or 1
|
||||
|
||||
# Memory
|
||||
mem_gb = 0
|
||||
try:
|
||||
# Linux: read from /proc/meminfo
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('MemTotal:'):
|
||||
mem_kb = int(line.split()[1])
|
||||
mem_gb = mem_kb / (1024 * 1024)
|
||||
break
|
||||
except (FileNotFoundError, PermissionError):
|
||||
# macOS: use sysctl
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['sysctl', '-n', 'hw.memsize'],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
mem_gb = int(result.stdout.strip()) / (1024**3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Disk space (current directory)
|
||||
disk_gb = 0
|
||||
try:
|
||||
statvfs = os.statvfs('.')
|
||||
disk_gb = (statvfs.f_frsize * statvfs.f_bavail) / (1024**3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
details = f"CPUs: {cpu_count}, Memory: {mem_gb:.1f}GB, Disk: {disk_gb:.1f}GB available"
|
||||
|
||||
# Check minimums
|
||||
warnings = []
|
||||
if cpu_count < 4:
|
||||
warnings.append(f"Low CPU count ({cpu_count}). Consider --max_cpus {cpu_count}")
|
||||
if 0 < mem_gb < 8:
|
||||
warnings.append(f"Low memory ({mem_gb:.1f}GB). Use --max_memory '{int(mem_gb)}GB'")
|
||||
if 0 < disk_gb < 50:
|
||||
warnings.append(f"Low disk space ({disk_gb:.1f}GB). Pipelines need ~100GB for human data")
|
||||
|
||||
if warnings:
|
||||
return CheckResult(
|
||||
name="Resources",
|
||||
passed=True,
|
||||
message="Resources available (with warnings)",
|
||||
details=details,
|
||||
fix="; ".join(warnings)
|
||||
)
|
||||
|
||||
return CheckResult(
|
||||
name="Resources",
|
||||
passed=True,
|
||||
message="Sufficient resources available",
|
||||
details=details
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return CheckResult(
|
||||
name="Resources",
|
||||
passed=True, # Don't fail on resource check errors
|
||||
message=f"Could not fully check resources: {str(e)}"
|
||||
)
|
||||
|
||||
|
||||
def check_network() -> CheckResult:
|
||||
"""Check network connectivity to Docker Hub and nf-core."""
|
||||
try:
|
||||
import urllib.request
|
||||
|
||||
# User-Agent header to avoid 403 from sites that block default Python agent
|
||||
headers = {'User-Agent': 'nf-core-helper/1.0'}
|
||||
|
||||
# Try Docker Hub
|
||||
try:
|
||||
req = urllib.request.Request("https://hub.docker.com", headers=headers)
|
||||
urllib.request.urlopen(req, timeout=10)
|
||||
docker_hub_ok = True
|
||||
except Exception:
|
||||
docker_hub_ok = False
|
||||
|
||||
# Try nf-core (for pipeline downloads)
|
||||
try:
|
||||
req = urllib.request.Request("https://nf-co.re", headers=headers)
|
||||
urllib.request.urlopen(req, timeout=10)
|
||||
nfcore_ok = True
|
||||
except Exception:
|
||||
nfcore_ok = False
|
||||
|
||||
if docker_hub_ok and nfcore_ok:
|
||||
return CheckResult(
|
||||
name="Network",
|
||||
passed=True,
|
||||
message="Network connectivity OK (Docker Hub & nf-core reachable)"
|
||||
)
|
||||
elif docker_hub_ok:
|
||||
return CheckResult(
|
||||
name="Network",
|
||||
passed=True,
|
||||
message="Docker Hub reachable (nf-core.re not reachable)",
|
||||
details="Pipeline downloads may still work via GitHub"
|
||||
)
|
||||
else:
|
||||
return CheckResult(
|
||||
name="Network",
|
||||
passed=False,
|
||||
message="Cannot reach Docker Hub",
|
||||
fix="Check network connection. Containers require Docker Hub access."
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
return CheckResult(
|
||||
name="Network",
|
||||
passed=False,
|
||||
message=f"Network check failed: {str(e)}",
|
||||
fix="Check network connection and proxy settings"
|
||||
)
|
||||
|
||||
|
||||
def run_all_checks() -> EnvironmentReport:
|
||||
"""Run all environment checks and return comprehensive report."""
|
||||
checks = [
|
||||
check_docker(),
|
||||
check_nextflow(),
|
||||
check_java(),
|
||||
check_resources(),
|
||||
check_network(),
|
||||
]
|
||||
|
||||
# Critical checks that must pass
|
||||
critical_checks = ["Docker", "Nextflow", "Java"]
|
||||
ready = all(c.passed for c in checks if c.name in critical_checks)
|
||||
|
||||
# Build recommendations
|
||||
recommendations = []
|
||||
for check in checks:
|
||||
if not check.passed and check.fix:
|
||||
recommendations.append(f"{check.name}: {check.fix}")
|
||||
elif check.passed and check.fix: # Warnings
|
||||
recommendations.append(f"{check.name} (warning): {check.fix}")
|
||||
|
||||
return EnvironmentReport(
|
||||
ready=ready,
|
||||
checks=checks,
|
||||
recommendations=recommendations
|
||||
)
|
||||
|
||||
|
||||
def print_report(report: EnvironmentReport):
|
||||
"""Print human-readable report to stdout."""
|
||||
print("\n" + "=" * 50)
|
||||
print(" nf-core Environment Check")
|
||||
print("=" * 50 + "\n")
|
||||
|
||||
for check in report.checks:
|
||||
status = "\033[92m[PASS]\033[0m" if check.passed else "\033[91m[FAIL]\033[0m"
|
||||
print(f"{status} {check.name}: {check.message}")
|
||||
|
||||
if check.details:
|
||||
print(f" {check.details}")
|
||||
|
||||
if not check.passed and check.fix:
|
||||
print(f" \033[93mFix:\033[0m {check.fix}")
|
||||
elif check.passed and check.fix: # Warning
|
||||
print(f" \033[93mWarning:\033[0m {check.fix}")
|
||||
|
||||
print()
|
||||
if report.ready:
|
||||
print("\033[92m✓ Environment is READY for nf-core pipelines.\033[0m")
|
||||
else:
|
||||
print("\033[91m✗ Environment is NOT READY. Please address the issues above.\033[0m")
|
||||
|
||||
if report.recommendations:
|
||||
print("\n--- Recommendations ---")
|
||||
for i, rec in enumerate(report.recommendations, 1):
|
||||
print(f" {i}. {rec}")
|
||||
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Check environment for nf-core pipeline execution",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python check_environment.py # Human-readable output
|
||||
python check_environment.py --json # JSON output for parsing
|
||||
"""
|
||||
)
|
||||
parser.add_argument("--json", action="store_true",
|
||||
help="Output results as JSON")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
report = run_all_checks()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(report.to_dict(), indent=2))
|
||||
else:
|
||||
print_report(report)
|
||||
|
||||
sys.exit(0 if report.ready else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,148 @@
|
||||
# Organism to Genome Mapping for nf-core Pipelines
|
||||
# Maps organism names (as they appear in GEO/SRA) to iGenomes keys
|
||||
|
||||
organisms:
|
||||
# Human
|
||||
"Homo sapiens":
|
||||
genome: "GRCh38"
|
||||
taxid: 9606
|
||||
aliases: ["human", "hg38", "GRCh38"]
|
||||
notes: "Primary human reference genome"
|
||||
|
||||
"Homo sapiens (legacy)":
|
||||
genome: "GRCh37"
|
||||
taxid: 9606
|
||||
aliases: ["hg19", "GRCh37"]
|
||||
notes: "Legacy human reference, still used for some clinical data"
|
||||
|
||||
# Mouse
|
||||
"Mus musculus":
|
||||
genome: "GRCm39"
|
||||
taxid: 10090
|
||||
aliases: ["mouse", "mm39", "GRCm39"]
|
||||
notes: "Current mouse reference genome"
|
||||
|
||||
"Mus musculus (legacy)":
|
||||
genome: "GRCm38"
|
||||
taxid: 10090
|
||||
aliases: ["mm10", "GRCm38"]
|
||||
notes: "Legacy mouse reference"
|
||||
|
||||
# Yeast
|
||||
"Saccharomyces cerevisiae":
|
||||
genome: "R64-1-1"
|
||||
taxid: 4932
|
||||
aliases: ["yeast", "sacCer3", "S288C", "budding yeast"]
|
||||
notes: "S288C reference strain"
|
||||
|
||||
# Fruit fly
|
||||
"Drosophila melanogaster":
|
||||
genome: "BDGP6"
|
||||
taxid: 7227
|
||||
aliases: ["fly", "dm6", "fruit fly", "Dmel"]
|
||||
notes: "Berkeley Drosophila Genome Project release 6"
|
||||
|
||||
# Worm
|
||||
"Caenorhabditis elegans":
|
||||
genome: "WBcel235"
|
||||
taxid: 6239
|
||||
aliases: ["worm", "ce11", "C. elegans", "Cele"]
|
||||
notes: "WormBase reference"
|
||||
|
||||
# Zebrafish
|
||||
"Danio rerio":
|
||||
genome: "GRCz11"
|
||||
taxid: 7955
|
||||
aliases: ["zebrafish", "danRer11", "Drer"]
|
||||
notes: "Genome Reference Consortium Zebrafish Build 11"
|
||||
|
||||
# Arabidopsis
|
||||
"Arabidopsis thaliana":
|
||||
genome: "TAIR10"
|
||||
taxid: 3702
|
||||
aliases: ["arabidopsis", "thale cress", "Atha"]
|
||||
notes: "The Arabidopsis Information Resource v10"
|
||||
|
||||
# Rat
|
||||
"Rattus norvegicus":
|
||||
genome: "Rnor_6.0"
|
||||
taxid: 10116
|
||||
aliases: ["rat", "rn6", "Rnor"]
|
||||
notes: "Rnor 6.0 reference"
|
||||
|
||||
# Chicken
|
||||
"Gallus gallus":
|
||||
genome: "GRCg6a"
|
||||
taxid: 9031
|
||||
aliases: ["chicken", "galGal6", "Ggal"]
|
||||
notes: "Genome Reference Consortium Chicken Build 6a"
|
||||
|
||||
# Pig
|
||||
"Sus scrofa":
|
||||
genome: "Sscrofa11.1"
|
||||
taxid: 9823
|
||||
aliases: ["pig", "susScr11", "Sscr"]
|
||||
notes: "Swine genome assembly 11.1"
|
||||
|
||||
# Cow
|
||||
"Bos taurus":
|
||||
genome: "ARS-UCD1.2"
|
||||
taxid: 9913
|
||||
aliases: ["cow", "bosTau9", "cattle", "Btau"]
|
||||
notes: "USDA ARS assembly"
|
||||
|
||||
# Dog
|
||||
"Canis lupus familiaris":
|
||||
genome: "CanFam3.1"
|
||||
taxid: 9615
|
||||
aliases: ["dog", "canFam3", "Clup"]
|
||||
notes: "Broad Institute CanFam3.1"
|
||||
|
||||
# Frog
|
||||
"Xenopus tropicalis":
|
||||
genome: "JGI_4.2"
|
||||
taxid: 8364
|
||||
aliases: ["frog", "xenTro9", "Xtro"]
|
||||
notes: "JGI assembly version 4.2"
|
||||
|
||||
# Maize/Corn
|
||||
"Zea mays":
|
||||
genome: "Zm-B73-REFERENCE-NAM-5.0"
|
||||
taxid: 4577
|
||||
aliases: ["maize", "corn", "Zmay"]
|
||||
notes: "B73 reference genome v5"
|
||||
|
||||
# Rice
|
||||
"Oryza sativa":
|
||||
genome: "IRGSP-1.0"
|
||||
taxid: 39947
|
||||
aliases: ["rice", "Osat"]
|
||||
notes: "International Rice Genome Sequencing Project"
|
||||
|
||||
# E. coli (common bacterial model)
|
||||
"Escherichia coli":
|
||||
genome: null
|
||||
taxid: 562
|
||||
aliases: ["E. coli", "Ecol"]
|
||||
notes: "Use specific strain reference; K-12 MG1655 common"
|
||||
|
||||
# Fission yeast
|
||||
"Schizosaccharomyces pombe":
|
||||
genome: "ASM294v2"
|
||||
taxid: 4896
|
||||
aliases: ["fission yeast", "S. pombe", "Spom"]
|
||||
notes: "PomBase reference"
|
||||
|
||||
# Pipeline mapping based on library strategy
|
||||
pipeline_suggestions:
|
||||
"RNA-SEQ": "rnaseq"
|
||||
"ATAC-SEQ": "atacseq"
|
||||
"CHIP-SEQ": "chipseq"
|
||||
"WGS": "sarek"
|
||||
"WXS": "sarek"
|
||||
"EXOME": "sarek"
|
||||
"AMPLICON": "ampliseq"
|
||||
"BISULFITE-SEQ": "methylseq"
|
||||
"HI-C": "hic"
|
||||
"MIRNA-SEQ": "smrnaseq"
|
||||
"RRBS": "methylseq"
|
||||
@@ -0,0 +1,187 @@
|
||||
name: atacseq
|
||||
version: "2.1.2"
|
||||
description: "Chromatin accessibility analysis and peak calling"
|
||||
|
||||
# Documentation and source - NOTE: Update version in URLs when upgrading pipeline
|
||||
urls:
|
||||
documentation: "https://nf-co.re/atacseq/{version}/"
|
||||
parameters: "https://nf-co.re/atacseq/{version}/parameters/"
|
||||
output_docs: "https://nf-co.re/atacseq/{version}/docs/output/"
|
||||
github: "https://github.com/nf-core/atacseq"
|
||||
releases: "https://github.com/nf-core/atacseq/releases"
|
||||
|
||||
data_types:
|
||||
- ATAC-seq
|
||||
- chromatin accessibility
|
||||
- open chromatin
|
||||
|
||||
detection_hints:
|
||||
filename:
|
||||
- atac
|
||||
- atacseq
|
||||
- chromatin
|
||||
- accessibility
|
||||
directory:
|
||||
- atac
|
||||
- atacseq
|
||||
- chromatin
|
||||
- epigenome
|
||||
- epigenetics
|
||||
|
||||
samplesheet:
|
||||
input_types:
|
||||
- fastq
|
||||
|
||||
columns:
|
||||
- name: sample
|
||||
required: true
|
||||
type: string
|
||||
inference: filename
|
||||
description: "Condition/group identifier (replicates share same name)"
|
||||
|
||||
- name: fastq_1
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
description: "Absolute path to R1 FASTQ"
|
||||
|
||||
- name: fastq_2
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
description: "Absolute path to R2 FASTQ (paired-end required)"
|
||||
|
||||
- name: replicate
|
||||
required: true
|
||||
type: integer
|
||||
default: 1
|
||||
inference: filename
|
||||
description: "Replicate number (integer)"
|
||||
|
||||
decision_points:
|
||||
- parameter: genome
|
||||
prompt: "Which reference genome matches your organism?"
|
||||
options:
|
||||
- value: GRCh38
|
||||
label: "Human GRCh38/hg38 (recommended)"
|
||||
description: "Latest human reference"
|
||||
- value: GRCh37
|
||||
label: "Human GRCh37/hg19 (legacy)"
|
||||
description: "Older human reference"
|
||||
- value: mm10
|
||||
label: "Mouse mm10"
|
||||
description: "Mouse reference genome"
|
||||
default: GRCh38
|
||||
recommendation: "Default to GRCh38 for human samples"
|
||||
|
||||
- parameter: read_length
|
||||
prompt: "What is the read length of your sequencing data?"
|
||||
options:
|
||||
- value: 50
|
||||
label: "50 bp"
|
||||
description: "Short reads"
|
||||
- value: 75
|
||||
label: "75 bp"
|
||||
description: "Standard length"
|
||||
- value: 100
|
||||
label: "100 bp"
|
||||
description: "Common for modern sequencers"
|
||||
- value: 150
|
||||
label: "150 bp"
|
||||
description: "Long reads"
|
||||
default: 50
|
||||
recommendation: "Check FASTQ files or sequencing report for exact length"
|
||||
|
||||
- parameter: narrow_peak
|
||||
prompt: "What type of peaks are you expecting?"
|
||||
options:
|
||||
- value: "true"
|
||||
label: "Narrow peaks (default for ATAC-seq)"
|
||||
description: "Standard ATAC-seq open chromatin regions"
|
||||
- value: "false"
|
||||
label: "Broad peaks"
|
||||
description: "For histone marks or broader accessibility regions"
|
||||
default: "true"
|
||||
recommendation: "Use narrow peaks for standard ATAC-seq"
|
||||
|
||||
test_profile:
|
||||
command: "nextflow run nf-core/atacseq -r 2.1.2 -profile test,docker --outdir test_atacseq"
|
||||
duration: "15 minutes"
|
||||
success_indicators:
|
||||
- "test_atacseq/multiqc/multiqc_report.html"
|
||||
log_pattern: "Pipeline completed successfully"
|
||||
|
||||
run_command:
|
||||
template: |
|
||||
nextflow run nf-core/atacseq \
|
||||
-r 2.1.2 \
|
||||
-profile docker \
|
||||
--input {samplesheet} \
|
||||
--outdir {outdir} \
|
||||
--genome {genome} \
|
||||
--read_length {read_length} \
|
||||
-resume
|
||||
|
||||
outputs:
|
||||
primary:
|
||||
- path: "bwa/mergedLibrary/*.mLb.mkD.sorted.bam"
|
||||
description: "Filtered, deduplicated alignments"
|
||||
- path: "bwa/mergedLibrary/bigwig/*.bigWig"
|
||||
description: "Coverage tracks for genome browsers"
|
||||
- path: "macs2/narrowPeak/*.narrowPeak"
|
||||
description: "Peak calls (BED format)"
|
||||
- path: "macs2/narrowPeak/consensus/consensus_peaks.bed"
|
||||
description: "Consensus peaks across replicates"
|
||||
|
||||
validation:
|
||||
- file: "multiqc/multiqc_report.html"
|
||||
check: exists
|
||||
description: "QC report must exist"
|
||||
- file: "macs2/narrowPeak"
|
||||
check: exists
|
||||
description: "Peak calls directory"
|
||||
|
||||
quality_metrics:
|
||||
- name: mapped_reads
|
||||
good: ">80%"
|
||||
acceptable: "60-80%"
|
||||
poor: "<60%"
|
||||
- name: mitochondrial
|
||||
good: "<20%"
|
||||
acceptable: "20-40%"
|
||||
poor: ">40%"
|
||||
- name: duplicates
|
||||
good: "<30%"
|
||||
acceptable: "30-50%"
|
||||
poor: ">50%"
|
||||
- name: frip
|
||||
good: ">30%"
|
||||
acceptable: "15-30%"
|
||||
poor: "<15%"
|
||||
- name: tss_enrichment
|
||||
good: ">6"
|
||||
acceptable: "4-6"
|
||||
poor: "<4"
|
||||
|
||||
resources:
|
||||
min_memory: "8.GB"
|
||||
recommended_memory: "32.GB"
|
||||
min_cpus: 4
|
||||
recommended_cpus: 8
|
||||
disk_space: "100.GB"
|
||||
|
||||
troubleshooting:
|
||||
- error: "Low FRiP score"
|
||||
fix: "Check library complexity in plotFingerprint. May indicate over-transposition or low quality"
|
||||
- error: "Few peaks called"
|
||||
fix: "Lower threshold with --macs_qvalue 0.1 or use --narrow_peak false for broader peaks"
|
||||
- error: "High duplicates"
|
||||
fix: "Normal for low-input samples. Pipeline removes by default. Consider deeper sequencing"
|
||||
- error: "High mitochondrial reads"
|
||||
fix: "Sample quality issue. Pipeline filters mito by default (--keep_mito false)"
|
||||
|
||||
replicate_patterns:
|
||||
- "_rep(\\d+)"
|
||||
- "_R(\\d+)_"
|
||||
- "_(\\d+)$"
|
||||
- "_replicate(\\d+)"
|
||||
@@ -0,0 +1,147 @@
|
||||
name: rnaseq
|
||||
version: "3.22.2"
|
||||
description: "Gene expression quantification and differential expression analysis"
|
||||
|
||||
# Documentation and source - NOTE: Update version in URLs when upgrading pipeline
|
||||
urls:
|
||||
documentation: "https://nf-co.re/rnaseq/{version}/"
|
||||
parameters: "https://nf-co.re/rnaseq/{version}/parameters/"
|
||||
output_docs: "https://nf-co.re/rnaseq/{version}/docs/output/"
|
||||
github: "https://github.com/nf-core/rnaseq"
|
||||
releases: "https://github.com/nf-core/rnaseq/releases"
|
||||
|
||||
data_types:
|
||||
- RNA-seq
|
||||
- mRNA-seq
|
||||
- bulk RNA-seq
|
||||
|
||||
detection_hints:
|
||||
filename:
|
||||
- rna
|
||||
- rnaseq
|
||||
- mrna
|
||||
- expression
|
||||
directory:
|
||||
- rnaseq
|
||||
- rna
|
||||
- expression
|
||||
- transcriptome
|
||||
|
||||
samplesheet:
|
||||
input_types:
|
||||
- fastq
|
||||
|
||||
columns:
|
||||
- name: sample
|
||||
required: true
|
||||
type: string
|
||||
inference: filename
|
||||
description: "Sample identifier"
|
||||
|
||||
- name: fastq_1
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
description: "Absolute path to R1 FASTQ"
|
||||
|
||||
- name: fastq_2
|
||||
required: false
|
||||
type: path
|
||||
inference: auto
|
||||
description: "Absolute path to R2 FASTQ (empty for single-end)"
|
||||
|
||||
- name: strandedness
|
||||
required: true
|
||||
type: enum
|
||||
allowed:
|
||||
- auto
|
||||
- forward
|
||||
- reverse
|
||||
- unstranded
|
||||
default: "auto"
|
||||
inference: default
|
||||
description: "Library strandedness (auto recommended)"
|
||||
|
||||
decision_points:
|
||||
- parameter: genome
|
||||
prompt: "Which reference genome matches your organism?"
|
||||
options:
|
||||
- value: GRCh38
|
||||
label: "Human GRCh38/hg38 (recommended for human)"
|
||||
description: "Latest human reference assembly"
|
||||
- value: GRCh37
|
||||
label: "Human GRCh37/hg19 (legacy)"
|
||||
description: "Older human reference for compatibility"
|
||||
- value: mm10
|
||||
label: "Mouse mm10/GRCm38"
|
||||
description: "Mouse reference genome"
|
||||
- value: BDGP6
|
||||
label: "Drosophila BDGP6"
|
||||
description: "Fruit fly reference"
|
||||
default: GRCh38
|
||||
recommendation: "Default to GRCh38 for human samples"
|
||||
|
||||
- parameter: aligner
|
||||
prompt: "Which alignment strategy would you prefer?"
|
||||
options:
|
||||
- value: star_salmon
|
||||
label: "STAR + Salmon (recommended)"
|
||||
description: "Most accurate, standard for differential expression"
|
||||
- value: star_rsem
|
||||
label: "STAR + RSEM"
|
||||
description: "Better for isoform-level quantification"
|
||||
- value: hisat2
|
||||
label: "HISAT2"
|
||||
description: "Lower memory requirements, faster"
|
||||
default: star_salmon
|
||||
recommendation: "Use star_salmon unless memory-constrained or need isoforms"
|
||||
|
||||
test_profile:
|
||||
command: "nextflow run nf-core/rnaseq -r 3.22.2 -profile test,docker --outdir test_rnaseq"
|
||||
duration: "15 minutes"
|
||||
success_indicators:
|
||||
- "test_rnaseq/multiqc/multiqc_report.html"
|
||||
log_pattern: "Pipeline completed successfully"
|
||||
|
||||
run_command:
|
||||
template: |
|
||||
nextflow run nf-core/rnaseq \
|
||||
-r 3.22.2 \
|
||||
-profile docker \
|
||||
--input {samplesheet} \
|
||||
--outdir {outdir} \
|
||||
--genome {genome} \
|
||||
--aligner {aligner} \
|
||||
-resume
|
||||
|
||||
outputs:
|
||||
primary:
|
||||
- path: "star_salmon/salmon.merged.gene_counts.tsv"
|
||||
description: "Raw gene counts for DESeq2/edgeR"
|
||||
- path: "star_salmon/salmon.merged.gene_tpm.tsv"
|
||||
description: "TPM normalized expression values"
|
||||
- path: "star_salmon/*.bam"
|
||||
description: "Aligned reads"
|
||||
|
||||
validation:
|
||||
- file: "multiqc/multiqc_report.html"
|
||||
check: exists
|
||||
description: "QC report must exist"
|
||||
- file: "star_salmon/salmon.merged.gene_counts.tsv"
|
||||
check: non_empty
|
||||
description: "Count matrix must have data"
|
||||
|
||||
resources:
|
||||
min_memory: "8.GB"
|
||||
recommended_memory: "32.GB"
|
||||
min_cpus: 4
|
||||
recommended_cpus: 8
|
||||
disk_space: "100.GB"
|
||||
|
||||
troubleshooting:
|
||||
- error: "STAR index fails"
|
||||
fix: "Increase memory with --max_memory '64.GB' or provide pre-built --star_index"
|
||||
- error: "Low alignment rate"
|
||||
fix: "Verify genome matches species; check FastQC for adapter contamination"
|
||||
- error: "Strandedness detection fails"
|
||||
fix: "Specify explicitly with --strandedness reverse (or forward/unstranded)"
|
||||
@@ -0,0 +1,233 @@
|
||||
name: sarek
|
||||
version: "3.7.1"
|
||||
description: "Variant calling for WGS/WES data (germline and somatic)"
|
||||
|
||||
# Documentation and source - NOTE: Update version in URLs when upgrading pipeline
|
||||
urls:
|
||||
documentation: "https://nf-co.re/sarek/{version}/"
|
||||
parameters: "https://nf-co.re/sarek/{version}/parameters/"
|
||||
output_docs: "https://nf-co.re/sarek/{version}/docs/output/"
|
||||
github: "https://github.com/nf-core/sarek"
|
||||
releases: "https://github.com/nf-core/sarek/releases"
|
||||
|
||||
data_types:
|
||||
- WGS
|
||||
- WES
|
||||
- whole genome sequencing
|
||||
- whole exome sequencing
|
||||
- tumor-normal
|
||||
- germline
|
||||
- somatic
|
||||
|
||||
detection_hints:
|
||||
filename:
|
||||
- tumor
|
||||
- normal
|
||||
- germline
|
||||
- wgs
|
||||
- wes
|
||||
- exome
|
||||
- dna
|
||||
- variant
|
||||
directory:
|
||||
- variant
|
||||
- wgs
|
||||
- wes
|
||||
- exome
|
||||
- germline
|
||||
- somatic
|
||||
|
||||
samplesheet:
|
||||
input_types:
|
||||
- fastq
|
||||
- bam
|
||||
- cram
|
||||
|
||||
columns:
|
||||
- name: patient
|
||||
required: true
|
||||
type: string
|
||||
inference: filename
|
||||
description: "Patient/subject identifier for grouping samples"
|
||||
|
||||
- name: sample
|
||||
required: true
|
||||
type: string
|
||||
inference: filename
|
||||
description: "Sample identifier (e.g., tumor, normal)"
|
||||
|
||||
- name: lane
|
||||
required: false
|
||||
type: string
|
||||
default: "L001"
|
||||
inference: filename
|
||||
description: "Sequencing lane"
|
||||
|
||||
- name: fastq_1
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
condition: "input_type == 'fastq'"
|
||||
description: "Absolute path to R1 FASTQ"
|
||||
|
||||
- name: fastq_2
|
||||
required: false
|
||||
type: path
|
||||
inference: auto
|
||||
condition: "input_type == 'fastq'"
|
||||
description: "Absolute path to R2 FASTQ"
|
||||
|
||||
- name: bam
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
condition: "input_type in ['bam', 'cram']"
|
||||
description: "Absolute path to BAM/CRAM file"
|
||||
|
||||
- name: bai
|
||||
required: true
|
||||
type: path
|
||||
inference: auto
|
||||
condition: "input_type in ['bam', 'cram']"
|
||||
description: "Absolute path to BAM/CRAM index"
|
||||
|
||||
- name: status
|
||||
required: false
|
||||
type: integer
|
||||
allowed:
|
||||
- 0
|
||||
- 1
|
||||
default: 0
|
||||
inference: filename
|
||||
description: "0=normal, 1=tumor (critical for somatic calling)"
|
||||
|
||||
decision_points:
|
||||
- parameter: genome
|
||||
prompt: "Which reference genome should be used?"
|
||||
options:
|
||||
- value: GRCh38
|
||||
label: "Human GRCh38/hg38 (recommended)"
|
||||
description: "Latest human reference with most annotation support"
|
||||
- value: GRCh37
|
||||
label: "Human GRCh37/hg19 (legacy)"
|
||||
description: "For compatibility with older datasets"
|
||||
- value: mm10
|
||||
label: "Mouse mm10"
|
||||
description: "Mouse reference genome"
|
||||
default: GRCh38
|
||||
recommendation: "Default to GRCh38 for human data"
|
||||
|
||||
- parameter: tools
|
||||
prompt: "What type of variant calling do you need?"
|
||||
options:
|
||||
- value: "haplotypecaller,snpeff"
|
||||
label: "Germline variants (single samples)"
|
||||
description: "For finding inherited variants in normal samples"
|
||||
condition: "no tumor samples detected"
|
||||
- value: "mutect2,strelka,snpeff"
|
||||
label: "Somatic variants (tumor-normal pairs)"
|
||||
description: "For finding cancer mutations with matched normal"
|
||||
condition: "tumor-normal pairs detected"
|
||||
- value: "haplotypecaller,deepvariant,snpeff"
|
||||
label: "Germline with DeepVariant"
|
||||
description: "Higher accuracy germline calling (requires GPU)"
|
||||
- value: "mutect2,manta,snpeff"
|
||||
label: "Somatic with structural variants"
|
||||
description: "Comprehensive tumor analysis including SVs"
|
||||
default: "haplotypecaller,snpeff"
|
||||
recommendation: "Use somatic tools if tumor/normal pairs detected, otherwise germline"
|
||||
|
||||
- parameter: wes
|
||||
prompt: "Is this whole exome sequencing (WES) data?"
|
||||
options:
|
||||
- value: "false"
|
||||
label: "No - Whole Genome Sequencing (WGS)"
|
||||
description: "Full genome coverage"
|
||||
- value: "true"
|
||||
label: "Yes - Whole Exome Sequencing (WES)"
|
||||
description: "Requires --intervals BED file"
|
||||
default: "false"
|
||||
recommendation: "If WES, user must provide intervals BED file"
|
||||
|
||||
test_profile:
|
||||
command: "nextflow run nf-core/sarek -r 3.7.1 -profile test,docker --outdir test_sarek"
|
||||
duration: "20 minutes"
|
||||
success_indicators:
|
||||
- "test_sarek/multiqc/multiqc_report.html"
|
||||
log_pattern: "Pipeline completed successfully"
|
||||
|
||||
run_command:
|
||||
template: |
|
||||
nextflow run nf-core/sarek \
|
||||
-r 3.7.1 \
|
||||
-profile docker \
|
||||
--input {samplesheet} \
|
||||
--outdir {outdir} \
|
||||
--genome {genome} \
|
||||
--tools {tools} \
|
||||
-resume
|
||||
|
||||
wes_template: |
|
||||
nextflow run nf-core/sarek \
|
||||
-r 3.7.1 \
|
||||
-profile docker \
|
||||
--input {samplesheet} \
|
||||
--outdir {outdir} \
|
||||
--genome {genome} \
|
||||
--tools {tools} \
|
||||
--wes \
|
||||
--intervals {intervals} \
|
||||
-resume
|
||||
|
||||
outputs:
|
||||
primary:
|
||||
- path: "preprocessing/recalibrated/*.recal.bam"
|
||||
description: "Analysis-ready BAM files"
|
||||
- path: "variant_calling/*/*.vcf.gz"
|
||||
description: "Variant call files"
|
||||
- path: "annotation/snpeff/*.ann.vcf.gz"
|
||||
description: "Annotated variants"
|
||||
|
||||
validation:
|
||||
- file: "multiqc/multiqc_report.html"
|
||||
check: exists
|
||||
description: "QC report must exist"
|
||||
- file: "preprocessing/recalibrated"
|
||||
check: exists
|
||||
description: "Recalibrated BAMs directory"
|
||||
|
||||
resources:
|
||||
min_memory: "16.GB"
|
||||
recommended_memory: "64.GB"
|
||||
wgs_memory: "128.GB"
|
||||
min_cpus: 4
|
||||
recommended_cpus: 16
|
||||
disk_space: "500.GB"
|
||||
|
||||
troubleshooting:
|
||||
- error: "BQSR fails"
|
||||
fix: "Check known sites available for genome. Skip with --skip_bqsr for non-standard references"
|
||||
- error: "Mutect2 no variants"
|
||||
fix: "Verify tumor/normal pairing in samplesheet (check status column: 0=normal, 1=tumor)"
|
||||
- error: "Out of memory"
|
||||
fix: "--max_memory '128.GB' for WGS data"
|
||||
- error: "DeepVariant GPU issues"
|
||||
fix: "Ensure NVIDIA Docker runtime configured, or use CPU mode"
|
||||
|
||||
tumor_normal_keywords:
|
||||
tumor:
|
||||
- tumor
|
||||
- tumour
|
||||
- met
|
||||
- metastasis
|
||||
- primary
|
||||
- cancer
|
||||
- malignant
|
||||
normal:
|
||||
- normal
|
||||
- germline
|
||||
- blood
|
||||
- pbmc
|
||||
- control
|
||||
- healthy
|
||||
- matched
|
||||
@@ -0,0 +1,300 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Auto-detect appropriate nf-core pipeline from data directory.
|
||||
|
||||
Analyzes filenames, directory structure, and file content hints to suggest
|
||||
the most appropriate pipeline for the data.
|
||||
|
||||
Usage:
|
||||
python detect_data_type.py /path/to/data
|
||||
python detect_data_type.py /path/to/data --json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def load_all_pipeline_configs() -> Dict[str, Dict]:
|
||||
"""Load all pipeline configurations."""
|
||||
config_dir = Path(__file__).parent / "config" / "pipelines"
|
||||
configs = {}
|
||||
|
||||
for config_file in config_dir.glob("*.yaml"):
|
||||
if config_file.stem.startswith("_"):
|
||||
continue
|
||||
with open(config_file) as f:
|
||||
configs[config_file.stem] = yaml.safe_load(f)
|
||||
|
||||
return configs
|
||||
|
||||
|
||||
def scan_directory(directory: str) -> Dict:
|
||||
"""Scan directory and collect file information."""
|
||||
info = {
|
||||
'fastq_count': 0,
|
||||
'bam_count': 0,
|
||||
'cram_count': 0,
|
||||
'filenames': [],
|
||||
'directories': [],
|
||||
'total_size_gb': 0,
|
||||
}
|
||||
|
||||
directory = os.path.abspath(directory)
|
||||
|
||||
for root, dirs, files in os.walk(directory):
|
||||
# Collect directory names
|
||||
rel_root = os.path.relpath(root, directory)
|
||||
if rel_root != '.':
|
||||
info['directories'].append(rel_root.lower())
|
||||
|
||||
for filename in files:
|
||||
filename_lower = filename.lower()
|
||||
|
||||
# Count file types
|
||||
if any(filename_lower.endswith(ext) for ext in ['.fastq.gz', '.fq.gz', '.fastq', '.fq']):
|
||||
info['fastq_count'] += 1
|
||||
elif filename_lower.endswith('.bam'):
|
||||
info['bam_count'] += 1
|
||||
elif filename_lower.endswith('.cram'):
|
||||
info['cram_count'] += 1
|
||||
|
||||
# Collect filenames for pattern matching
|
||||
info['filenames'].append(filename_lower)
|
||||
|
||||
# Sum file sizes
|
||||
try:
|
||||
size = os.path.getsize(os.path.join(root, filename))
|
||||
info['total_size_gb'] += size / (1024**3)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def calculate_pipeline_scores(scan_info: Dict, configs: Dict) -> Dict[str, Dict]:
|
||||
"""Calculate confidence scores for each pipeline."""
|
||||
scores = {}
|
||||
|
||||
for pipeline_name, config in configs.items():
|
||||
score = 0
|
||||
matches = []
|
||||
|
||||
# Check detection hints
|
||||
hints = config.get('detection_hints', {})
|
||||
|
||||
# Filename hints
|
||||
filename_hints = hints.get('filename', [])
|
||||
for hint in filename_hints:
|
||||
hint_lower = hint.lower()
|
||||
for filename in scan_info['filenames']:
|
||||
if hint_lower in filename:
|
||||
score += 10
|
||||
matches.append(f"Filename contains '{hint}'")
|
||||
break
|
||||
|
||||
# Directory hints
|
||||
directory_hints = hints.get('directory', [])
|
||||
for hint in directory_hints:
|
||||
hint_lower = hint.lower()
|
||||
for dirname in scan_info['directories']:
|
||||
if hint_lower in dirname:
|
||||
score += 15
|
||||
matches.append(f"Directory contains '{hint}'")
|
||||
break
|
||||
|
||||
# Check data type compatibility
|
||||
data_types = config.get('data_types', [])
|
||||
input_types = config.get('samplesheet', {}).get('input_types', ['fastq'])
|
||||
|
||||
# Prefer pipelines that support the available file types
|
||||
if 'fastq' in input_types and scan_info['fastq_count'] > 0:
|
||||
score += 5
|
||||
if 'bam' in input_types and scan_info['bam_count'] > 0:
|
||||
score += 5
|
||||
if 'cram' in input_types and scan_info['cram_count'] > 0:
|
||||
score += 5
|
||||
|
||||
# Pipeline-specific boosts
|
||||
if pipeline_name == 'sarek':
|
||||
# Check for tumor/normal indicators
|
||||
tumor_indicators = ['tumor', 'tumour', 'cancer', 'met', 'primary']
|
||||
normal_indicators = ['normal', 'germline', 'blood', 'control']
|
||||
|
||||
has_tumor = any(ind in ' '.join(scan_info['filenames']) for ind in tumor_indicators)
|
||||
has_normal = any(ind in ' '.join(scan_info['filenames']) for ind in normal_indicators)
|
||||
|
||||
if has_tumor or has_normal:
|
||||
score += 20
|
||||
if has_tumor:
|
||||
matches.append("Found tumor sample indicators")
|
||||
if has_normal:
|
||||
matches.append("Found normal sample indicators")
|
||||
|
||||
# DNA-related hints
|
||||
dna_hints = ['wgs', 'wes', 'exome', 'dna', 'variant', 'snp', 'indel']
|
||||
for hint in dna_hints:
|
||||
if hint in ' '.join(scan_info['filenames'] + scan_info['directories']):
|
||||
score += 10
|
||||
matches.append(f"Found DNA/variant indicator: '{hint}'")
|
||||
break
|
||||
|
||||
elif pipeline_name == 'rnaseq':
|
||||
# RNA-related hints
|
||||
rna_hints = ['rna', 'rnaseq', 'mrna', 'expression', 'transcript', 'counts']
|
||||
for hint in rna_hints:
|
||||
if hint in ' '.join(scan_info['filenames'] + scan_info['directories']):
|
||||
score += 15
|
||||
matches.append(f"Found RNA indicator: '{hint}'")
|
||||
break
|
||||
|
||||
elif pipeline_name == 'atacseq':
|
||||
# ATAC-related hints
|
||||
atac_hints = ['atac', 'atacseq', 'chromatin', 'accessibility', 'peak', 'macs']
|
||||
for hint in atac_hints:
|
||||
if hint in ' '.join(scan_info['filenames'] + scan_info['directories']):
|
||||
score += 20
|
||||
matches.append(f"Found ATAC-seq indicator: '{hint}'")
|
||||
break
|
||||
|
||||
scores[pipeline_name] = {
|
||||
'score': score,
|
||||
'matches': matches,
|
||||
'description': config.get('description', ''),
|
||||
'version': config.get('version', 'unknown'),
|
||||
}
|
||||
|
||||
return scores
|
||||
|
||||
|
||||
def detect_pipeline(directory: str) -> Tuple[str, Dict]:
|
||||
"""
|
||||
Detect the most appropriate pipeline for the data.
|
||||
|
||||
Args:
|
||||
directory: Path to data directory
|
||||
|
||||
Returns:
|
||||
Tuple of (recommended_pipeline, all_scores)
|
||||
"""
|
||||
if not os.path.isdir(directory):
|
||||
raise ValueError(f"Not a directory: {directory}")
|
||||
|
||||
configs = load_all_pipeline_configs()
|
||||
scan_info = scan_directory(directory)
|
||||
|
||||
# Check if any sequencing files found
|
||||
total_files = scan_info['fastq_count'] + scan_info['bam_count'] + scan_info['cram_count']
|
||||
if total_files == 0:
|
||||
raise ValueError(f"No sequencing files (FASTQ/BAM/CRAM) found in {directory}")
|
||||
|
||||
scores = calculate_pipeline_scores(scan_info, configs)
|
||||
|
||||
# Find highest scoring pipeline
|
||||
best_pipeline = max(scores.keys(), key=lambda k: scores[k]['score'])
|
||||
|
||||
return best_pipeline, scores
|
||||
|
||||
|
||||
def print_results(
|
||||
directory: str,
|
||||
recommended: str,
|
||||
scores: Dict,
|
||||
scan_info: Dict,
|
||||
output_json: bool = False
|
||||
):
|
||||
"""Print detection results."""
|
||||
if output_json:
|
||||
result = {
|
||||
'recommended': recommended,
|
||||
'scores': scores,
|
||||
'scan_info': {
|
||||
'fastq_count': scan_info['fastq_count'],
|
||||
'bam_count': scan_info['bam_count'],
|
||||
'cram_count': scan_info['cram_count'],
|
||||
'total_size_gb': round(scan_info['total_size_gb'], 2),
|
||||
}
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
return
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print(" nf-core Pipeline Detection")
|
||||
print("=" * 50)
|
||||
print(f"\nDirectory: {directory}")
|
||||
print(f"Files found: {scan_info['fastq_count']} FASTQ, "
|
||||
f"{scan_info['bam_count']} BAM, {scan_info['cram_count']} CRAM")
|
||||
print(f"Total size: {scan_info['total_size_gb']:.1f} GB")
|
||||
|
||||
print("\n--- Pipeline Scores ---")
|
||||
sorted_pipelines = sorted(scores.keys(), key=lambda k: scores[k]['score'], reverse=True)
|
||||
|
||||
for pipeline in sorted_pipelines:
|
||||
info = scores[pipeline]
|
||||
indicator = "→" if pipeline == recommended else " "
|
||||
print(f"\n{indicator} {pipeline} (score: {info['score']})")
|
||||
print(f" {info['description']}")
|
||||
if info['matches']:
|
||||
print(f" Matches: {', '.join(info['matches'][:3])}")
|
||||
|
||||
print(f"\n{'=' * 50}")
|
||||
print(f"\n\033[92mRecommended: {recommended}\033[0m")
|
||||
print(f"Version: {scores[recommended]['version']}")
|
||||
|
||||
# Print suggested next steps
|
||||
print(f"\n--- Next Steps ---")
|
||||
print(f"1. Run environment check:")
|
||||
print(f" python scripts/check_environment.py")
|
||||
print(f"\n2. Run test profile:")
|
||||
config = load_all_pipeline_configs().get(recommended, {})
|
||||
test_cmd = config.get('test_profile', {}).get('command', '')
|
||||
if test_cmd:
|
||||
print(f" {test_cmd}")
|
||||
print(f"\n3. Generate samplesheet:")
|
||||
print(f" python scripts/generate_samplesheet.py {directory} {recommended}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Detect appropriate nf-core pipeline for data',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s ./data
|
||||
%(prog)s ./fastqs --json
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('directory', help='Directory containing sequencing data')
|
||||
parser.add_argument('--json', action='store_true', help='Output as JSON')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
scan_info = scan_directory(args.directory)
|
||||
recommended, scores = detect_pipeline(args.directory)
|
||||
print_results(args.directory, recommended, scores, scan_info, args.json)
|
||||
sys.exit(0)
|
||||
|
||||
except ValueError as e:
|
||||
if args.json:
|
||||
print(json.dumps({'error': str(e)}))
|
||||
else:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
if args.json:
|
||||
print(json.dumps({'error': str(e)}))
|
||||
else:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,455 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Enhanced nf-core samplesheet generator.
|
||||
|
||||
Features:
|
||||
- FASTQ, BAM, and CRAM support
|
||||
- Tumor/normal status inference for sarek
|
||||
- Robust R1/R2 matching with scoring
|
||||
- Pre-write validation with clear error messages
|
||||
- Pipeline config-driven column generation
|
||||
|
||||
Usage:
|
||||
python generate_samplesheet.py /path/to/data rnaseq -o samplesheet.csv
|
||||
python generate_samplesheet.py /path/to/bams sarek --input-type bam
|
||||
python generate_samplesheet.py --validate samplesheet.csv rnaseq
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
import yaml
|
||||
|
||||
# Add parent directory to path for utils import
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
|
||||
from utils.file_discovery import discover_files, detect_input_type, find_index_file
|
||||
from utils.sample_inference import (
|
||||
extract_sample_info,
|
||||
infer_tumor_normal_status,
|
||||
match_read_pairs,
|
||||
extract_replicate_number
|
||||
)
|
||||
from utils.validators import validate_samplesheet, ValidationResult
|
||||
|
||||
|
||||
def load_pipeline_config(pipeline: str) -> Dict:
|
||||
"""Load pipeline configuration from YAML."""
|
||||
config_dir = Path(__file__).parent / "config" / "pipelines"
|
||||
config_file = config_dir / f"{pipeline}.yaml"
|
||||
|
||||
if not config_file.exists():
|
||||
available = [f.stem for f in config_dir.glob("*.yaml") if not f.stem.startswith("_")]
|
||||
raise ValueError(f"Unknown pipeline '{pipeline}'. Available: {', '.join(available)}")
|
||||
|
||||
with open(config_file) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def generate_samplesheet(
|
||||
input_dir: str,
|
||||
pipeline: str,
|
||||
output_file: Optional[str] = None,
|
||||
input_type: str = "auto",
|
||||
single_end: bool = False,
|
||||
interactive: bool = True
|
||||
) -> Tuple[Optional[str], ValidationResult]:
|
||||
"""
|
||||
Generate samplesheet for specified pipeline.
|
||||
|
||||
Args:
|
||||
input_dir: Directory containing sequencing files
|
||||
pipeline: Pipeline name (rnaseq, sarek, atacseq)
|
||||
output_file: Output CSV path (default: samplesheet_{pipeline}.csv)
|
||||
input_type: File type (auto, fastq, bam, cram)
|
||||
single_end: Suppress pairing warnings for single-end data
|
||||
interactive: Prompt for missing info
|
||||
|
||||
Returns:
|
||||
Tuple of (output_path, validation_result)
|
||||
"""
|
||||
config = load_pipeline_config(pipeline)
|
||||
samplesheet_config = config.get("samplesheet", {})
|
||||
supported_types = samplesheet_config.get("input_types", ["fastq"])
|
||||
|
||||
# Determine input type
|
||||
if input_type == "auto":
|
||||
input_type = detect_input_type(input_dir)
|
||||
print(f"Auto-detected input type: {input_type.upper()}")
|
||||
|
||||
if input_type not in supported_types:
|
||||
return None, ValidationResult(
|
||||
valid=False,
|
||||
errors=[f"Pipeline '{pipeline}' does not support {input_type.upper()} input. "
|
||||
f"Supported: {supported_types}"]
|
||||
)
|
||||
|
||||
# Discover files
|
||||
try:
|
||||
files = discover_files(input_dir, input_type)
|
||||
except ValueError as e:
|
||||
return None, ValidationResult(valid=False, errors=[str(e)])
|
||||
|
||||
if not files:
|
||||
return None, ValidationResult(
|
||||
valid=False,
|
||||
errors=[f"No {input_type.upper()} files found in {input_dir}"],
|
||||
suggestions=[
|
||||
"Check directory path is correct",
|
||||
"Verify file extensions (.fastq.gz, .fq.gz, .bam, .cram)",
|
||||
f"Run: ls {input_dir}"
|
||||
]
|
||||
)
|
||||
|
||||
print(f"Found {len(files)} {input_type.upper()} files")
|
||||
|
||||
# Process based on input type
|
||||
if input_type == "fastq":
|
||||
rows = _process_fastq_files(files, config, single_end)
|
||||
else:
|
||||
rows = _process_alignment_files(files, config, input_type)
|
||||
|
||||
if not rows:
|
||||
return None, ValidationResult(
|
||||
valid=False,
|
||||
errors=["Could not generate any samplesheet rows from files"]
|
||||
)
|
||||
|
||||
print(f"Generated {len(rows)} samplesheet rows")
|
||||
|
||||
# Pipeline-specific processing
|
||||
if pipeline == "sarek":
|
||||
rows = _process_sarek_samples(rows, interactive)
|
||||
elif pipeline == "atacseq":
|
||||
rows = _process_atacseq_samples(rows)
|
||||
|
||||
# Validate before writing
|
||||
validation = validate_samplesheet(rows, pipeline, config)
|
||||
|
||||
if not validation.valid:
|
||||
print("\nValidation errors:")
|
||||
for error in validation.errors:
|
||||
print(f" - {error}")
|
||||
|
||||
if interactive:
|
||||
response = input("\nProceed anyway? [y/N]: ").strip().lower()
|
||||
if response != 'y':
|
||||
return None, validation
|
||||
elif validation.warnings:
|
||||
print("\nWarnings:")
|
||||
for warning in validation.warnings:
|
||||
print(f" - {warning}")
|
||||
|
||||
# Determine output path
|
||||
output_path = output_file or f"samplesheet_{pipeline}.csv"
|
||||
|
||||
# Write samplesheet
|
||||
_write_samplesheet(rows, config, output_path)
|
||||
|
||||
print(f"\nGenerated: {output_path}")
|
||||
print(f" Pipeline: {pipeline} v{config.get('version', 'unknown')}")
|
||||
print(f" Samples: {len(set(r.get('sample', r.get('patient', '')) for r in rows))}")
|
||||
print(f" Rows: {len(rows)}")
|
||||
|
||||
# Preview
|
||||
_print_preview(rows, config)
|
||||
|
||||
return output_path, validation
|
||||
|
||||
|
||||
def _process_fastq_files(files, config: Dict, single_end: bool) -> List[Dict]:
|
||||
"""Process FASTQ files into samplesheet rows."""
|
||||
pairs = match_read_pairs(files)
|
||||
|
||||
if not pairs:
|
||||
return []
|
||||
|
||||
# Check for unpaired files
|
||||
unpaired = [k for k, v in pairs.items() if v.get('r1') and not v.get('r2')]
|
||||
if unpaired and not single_end:
|
||||
print(f"\nNote: {len(unpaired)} samples appear to be single-end (no R2)")
|
||||
|
||||
rows = []
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
|
||||
for sample_key, pair_info in sorted(pairs.items()):
|
||||
if not pair_info.get('r1'):
|
||||
continue # Skip entries with only R2
|
||||
|
||||
info = pair_info.get('info', {})
|
||||
|
||||
row = {
|
||||
'sample': info.get('sample', sample_key),
|
||||
'fastq_1': str(Path(pair_info['r1']).absolute()),
|
||||
'fastq_2': str(Path(pair_info['r2']).absolute()) if pair_info.get('r2') else '',
|
||||
}
|
||||
|
||||
# Add additional info from filename
|
||||
if 'patient' in [c['name'] for c in columns]:
|
||||
row['patient'] = info.get('patient', info.get('sample', sample_key))
|
||||
|
||||
if 'lane' in [c['name'] for c in columns]:
|
||||
row['lane'] = info.get('lane', 'L001')
|
||||
|
||||
# Apply defaults from config
|
||||
for col in columns:
|
||||
if col['name'] not in row and 'default' in col:
|
||||
row[col['name']] = col['default']
|
||||
|
||||
rows.append(row)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _process_alignment_files(files, config: Dict, input_type: str) -> List[Dict]:
|
||||
"""Process BAM/CRAM files into samplesheet rows."""
|
||||
rows = []
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
|
||||
for file_info in files:
|
||||
# Find index file
|
||||
index_path = find_index_file(file_info.path)
|
||||
|
||||
info = extract_sample_info(file_info.path)
|
||||
|
||||
row = {
|
||||
'sample': info.get('sample', file_info.stem),
|
||||
'bam': str(Path(file_info.path).absolute()),
|
||||
'bai': str(Path(index_path).absolute()) if index_path else '',
|
||||
}
|
||||
|
||||
# Add patient for sarek
|
||||
if 'patient' in [c['name'] for c in columns]:
|
||||
row['patient'] = info.get('patient', info.get('sample', file_info.stem))
|
||||
|
||||
# Apply defaults
|
||||
for col in columns:
|
||||
if col['name'] not in row and 'default' in col:
|
||||
row[col['name']] = col['default']
|
||||
|
||||
# Warn if no index found
|
||||
if not index_path:
|
||||
print(f" Warning: No index found for {file_info.name}")
|
||||
|
||||
rows.append(row)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _process_sarek_samples(rows: List[Dict], interactive: bool) -> List[Dict]:
|
||||
"""Process sarek samples: infer and confirm tumor/normal status."""
|
||||
# Auto-infer status from sample names
|
||||
for row in rows:
|
||||
sample_name = row.get('sample', '')
|
||||
inferred = infer_tumor_normal_status(sample_name)
|
||||
if inferred is not None:
|
||||
row['status'] = inferred
|
||||
|
||||
# Report inference results
|
||||
inferred_tumor = [r for r in rows if r.get('status') == 1]
|
||||
inferred_normal = [r for r in rows if r.get('status') == 0]
|
||||
unknown = [r for r in rows if 'status' not in r]
|
||||
|
||||
if inferred_tumor or inferred_normal:
|
||||
print(f"\nTumor/normal inference:")
|
||||
print(f" Tumor samples: {len(inferred_tumor)}")
|
||||
print(f" Normal samples: {len(inferred_normal)}")
|
||||
|
||||
# Handle unknown samples
|
||||
if unknown and interactive:
|
||||
print(f"\n{len(unknown)} sample(s) with unknown status:")
|
||||
for r in unknown:
|
||||
print(f" - {r.get('sample')}")
|
||||
|
||||
print("\nSpecify status for each (0=normal, 1=tumor, Enter=skip):")
|
||||
for r in unknown:
|
||||
response = input(f" {r.get('sample')} [0/1/Enter]: ").strip()
|
||||
if response in ['0', '1']:
|
||||
r['status'] = int(response)
|
||||
else:
|
||||
r['status'] = 0 # Default to normal
|
||||
print(f" Defaulting to normal (0)")
|
||||
elif unknown:
|
||||
# Non-interactive: default to normal
|
||||
for r in unknown:
|
||||
r['status'] = 0
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _process_atacseq_samples(rows: List[Dict]) -> List[Dict]:
|
||||
"""Process ATAC-seq samples: ensure replicate numbers."""
|
||||
# Group by sample name
|
||||
sample_counts = {}
|
||||
for row in rows:
|
||||
sample = row.get('sample', '')
|
||||
if sample not in sample_counts:
|
||||
sample_counts[sample] = 0
|
||||
sample_counts[sample] += 1
|
||||
|
||||
# Assign replicate numbers if not present
|
||||
sample_rep = {}
|
||||
for row in rows:
|
||||
sample = row.get('sample', '')
|
||||
|
||||
if 'replicate' not in row or not row['replicate']:
|
||||
# Try to extract from filename
|
||||
extracted = extract_replicate_number(row.get('fastq_1', ''))
|
||||
if extracted:
|
||||
row['replicate'] = extracted
|
||||
else:
|
||||
# Auto-assign sequential
|
||||
if sample not in sample_rep:
|
||||
sample_rep[sample] = 0
|
||||
sample_rep[sample] += 1
|
||||
row['replicate'] = sample_rep[sample]
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _write_samplesheet(rows: List[Dict], config: Dict, output_path: str):
|
||||
"""Write samplesheet to CSV file."""
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
column_names = [c['name'] for c in columns]
|
||||
|
||||
# Filter to columns that have data
|
||||
active_columns = [c for c in column_names if any(c in row and row[c] for row in rows)]
|
||||
|
||||
# Ensure fastq_1/fastq_2 or bam/bai are included
|
||||
for required in ['fastq_1', 'bam']:
|
||||
if required in column_names and required not in active_columns:
|
||||
if any(required in row for row in rows):
|
||||
active_columns.append(required)
|
||||
|
||||
# Maintain original column order
|
||||
active_columns = [c for c in column_names if c in active_columns]
|
||||
|
||||
with open(output_path, 'w') as f:
|
||||
f.write(','.join(active_columns) + '\n')
|
||||
for row in rows:
|
||||
values = [str(row.get(col, '')) for col in active_columns]
|
||||
f.write(','.join(values) + '\n')
|
||||
|
||||
|
||||
def _print_preview(rows: List[Dict], config: Dict):
|
||||
"""Print preview of generated samplesheet."""
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
column_names = [c['name'] for c in columns]
|
||||
active_columns = [c for c in column_names if any(c in row for row in rows)]
|
||||
|
||||
print(f"\nPreview (first 3 rows):")
|
||||
print(','.join(active_columns))
|
||||
for row in rows[:3]:
|
||||
values = [str(row.get(col, ''))[:40] for col in active_columns] # Truncate long paths
|
||||
print(','.join(values))
|
||||
if len(rows) > 3:
|
||||
print(f"... ({len(rows) - 3} more rows)")
|
||||
|
||||
|
||||
def validate_existing_samplesheet(csv_path: str, pipeline: str) -> ValidationResult:
|
||||
"""Validate an existing samplesheet file."""
|
||||
import csv
|
||||
|
||||
if not os.path.exists(csv_path):
|
||||
return ValidationResult(valid=False, errors=[f"File not found: {csv_path}"])
|
||||
|
||||
try:
|
||||
with open(csv_path, 'r') as f:
|
||||
reader = csv.DictReader(f)
|
||||
rows = list(reader)
|
||||
except Exception as e:
|
||||
return ValidationResult(valid=False, errors=[f"Failed to read CSV: {e}"])
|
||||
|
||||
if not rows:
|
||||
return ValidationResult(valid=False, errors=["Samplesheet is empty"])
|
||||
|
||||
config = load_pipeline_config(pipeline)
|
||||
return validate_samplesheet(rows, pipeline, config)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Generate nf-core samplesheet from data directory',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Generate samplesheet for RNA-seq
|
||||
%(prog)s ./fastqs rnaseq -o samples.csv
|
||||
|
||||
# Generate samplesheet for sarek from BAM files
|
||||
%(prog)s ./bams sarek --input-type bam
|
||||
|
||||
# Validate existing samplesheet
|
||||
%(prog)s --validate samplesheet.csv rnaseq
|
||||
|
||||
Supported pipelines: rnaseq, sarek, atacseq
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('input', help='Directory with data files, or CSV path for --validate')
|
||||
parser.add_argument('pipeline', help='Pipeline name (rnaseq, sarek, atacseq)')
|
||||
parser.add_argument('-o', '--output', help='Output CSV filename')
|
||||
parser.add_argument('--input-type', choices=['auto', 'fastq', 'bam', 'cram'],
|
||||
default='auto', help='Input file type (default: auto-detect)')
|
||||
parser.add_argument('--single-end', action='store_true',
|
||||
help='Treat as single-end data (suppress pairing warnings)')
|
||||
parser.add_argument('--validate', action='store_true',
|
||||
help='Validate existing samplesheet instead of generating')
|
||||
parser.add_argument('--no-interactive', action='store_true',
|
||||
help='Non-interactive mode (use defaults)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
if args.validate:
|
||||
# Validate existing samplesheet
|
||||
result = validate_existing_samplesheet(args.input, args.pipeline)
|
||||
if result.valid:
|
||||
print(f"✓ Samplesheet is valid for {args.pipeline}")
|
||||
if result.warnings:
|
||||
print("\nWarnings:")
|
||||
for w in result.warnings:
|
||||
print(f" - {w}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"✗ Samplesheet validation failed")
|
||||
print(result.summary())
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Generate new samplesheet
|
||||
if not os.path.isdir(args.input):
|
||||
print(f"Error: Not a directory: {args.input}")
|
||||
sys.exit(1)
|
||||
|
||||
output_path, result = generate_samplesheet(
|
||||
args.input,
|
||||
args.pipeline,
|
||||
args.output,
|
||||
args.input_type,
|
||||
args.single_end,
|
||||
interactive=not args.no_interactive
|
||||
)
|
||||
|
||||
if output_path is None:
|
||||
print("\nFailed to generate samplesheet.")
|
||||
if result.suggestions:
|
||||
print("\nSuggestions:")
|
||||
for s in result.suggestions:
|
||||
print(f" - {s}")
|
||||
sys.exit(1)
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
except ValueError as e:
|
||||
print(f"Error: {e}")
|
||||
sys.exit(1)
|
||||
except KeyboardInterrupt:
|
||||
print("\nAborted.")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,521 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Genome reference management for nf-core pipelines.
|
||||
|
||||
Manages downloading, caching, and accessing genome references from iGenomes.
|
||||
Supports auto-download when references aren't available locally.
|
||||
|
||||
Usage:
|
||||
python manage_genomes.py list
|
||||
python manage_genomes.py check GRCh38
|
||||
python manage_genomes.py download GRCh38
|
||||
python manage_genomes.py params GRCh38
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
# iGenomes reference configuration
|
||||
IGENOMES = {
|
||||
# Human
|
||||
'GRCh38': {
|
||||
'display_name': 'Human GRCh38/hg38',
|
||||
'species': 'Homo sapiens',
|
||||
'aliases': ['hg38', 'GRCh38.p14'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Homo_sapiens/NCBI/GRCh38',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
'GRCh37': {
|
||||
'display_name': 'Human GRCh37/hg19',
|
||||
'species': 'Homo sapiens',
|
||||
'aliases': ['hg19', 'GRCh37.p13'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Homo_sapiens/NCBI/GRCh37',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Mouse
|
||||
'GRCm39': {
|
||||
'display_name': 'Mouse GRCm39/mm39',
|
||||
'species': 'Mus musculus',
|
||||
'aliases': ['mm39', 'GRCm39'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Mus_musculus/Ensembl/GRCm39',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
'GRCm38': {
|
||||
'display_name': 'Mouse GRCm38/mm10',
|
||||
'species': 'Mus musculus',
|
||||
'aliases': ['mm10', 'GRCm38'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Mus_musculus/NCBI/GRCm38',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Yeast
|
||||
'R64-1-1': {
|
||||
'display_name': 'Yeast R64-1-1/sacCer3',
|
||||
'species': 'Saccharomyces cerevisiae',
|
||||
'aliases': ['sacCer3', 'S288C', 'yeast'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Saccharomyces_cerevisiae/Ensembl/R64-1-1',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Fruit fly
|
||||
'BDGP6': {
|
||||
'display_name': 'Drosophila BDGP6/dm6',
|
||||
'species': 'Drosophila melanogaster',
|
||||
'aliases': ['dm6', 'BDGP6', 'fly'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Drosophila_melanogaster/Ensembl/BDGP6',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
}
|
||||
},
|
||||
# C. elegans
|
||||
'WBcel235': {
|
||||
'display_name': 'C. elegans WBcel235/ce11',
|
||||
'species': 'Caenorhabditis elegans',
|
||||
'aliases': ['ce11', 'worm'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Caenorhabditis_elegans/Ensembl/WBcel235',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Zebrafish
|
||||
'GRCz11': {
|
||||
'display_name': 'Zebrafish GRCz11/danRer11',
|
||||
'species': 'Danio rerio',
|
||||
'aliases': ['danRer11', 'zebrafish'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz11',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
'GRCz10': {
|
||||
'display_name': 'Zebrafish GRCz10/danRer10',
|
||||
'species': 'Danio rerio',
|
||||
'aliases': ['danRer10'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz10',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
}
|
||||
},
|
||||
# Rat
|
||||
'Rnor_6.0': {
|
||||
'display_name': 'Rat Rnor_6.0/rn6',
|
||||
'species': 'Rattus norvegicus',
|
||||
'aliases': ['rn6', 'Rnor6', 'rat'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Rattus_norvegicus/Ensembl/Rnor_6.0',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Arabidopsis
|
||||
'TAIR10': {
|
||||
'display_name': 'Arabidopsis TAIR10',
|
||||
'species': 'Arabidopsis thaliana',
|
||||
'aliases': ['arabidopsis'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Arabidopsis_thaliana/Ensembl/TAIR10',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
'bwa_index': 'Sequence/BWAIndex/',
|
||||
'star_index': 'Sequence/STARIndex/',
|
||||
}
|
||||
},
|
||||
# Chicken
|
||||
'GRCg6a': {
|
||||
'display_name': 'Chicken GRCg6a/galGal6',
|
||||
'species': 'Gallus gallus',
|
||||
'aliases': ['galGal6', 'chicken'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Gallus_gallus/Ensembl/GRCg6a',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
}
|
||||
},
|
||||
# Dog
|
||||
'CanFam3.1': {
|
||||
'display_name': 'Dog CanFam3.1/canFam3',
|
||||
'species': 'Canis lupus familiaris',
|
||||
'aliases': ['canFam3', 'dog'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Canis_familiaris/Ensembl/CanFam3.1',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
}
|
||||
},
|
||||
# Pig
|
||||
'Sscrofa11.1': {
|
||||
'display_name': 'Pig Sscrofa11.1/susScr11',
|
||||
'species': 'Sus scrofa',
|
||||
'aliases': ['susScr11', 'pig'],
|
||||
's3_base': 's3://ngi-igenomes/igenomes/Sus_scrofa/Ensembl/Sscrofa11.1',
|
||||
'files': {
|
||||
'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
|
||||
'gtf': 'Annotation/Genes/genes.gtf',
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_cache_dir() -> Path:
|
||||
"""Get genome cache directory."""
|
||||
cache_dir = os.environ.get(
|
||||
'NF_CORE_GENOME_CACHE',
|
||||
os.path.expanduser('~/.nf-core/genomes')
|
||||
)
|
||||
return Path(cache_dir)
|
||||
|
||||
|
||||
def resolve_genome_id(genome: str) -> Optional[str]:
|
||||
"""Resolve genome ID from name or alias."""
|
||||
# Direct match
|
||||
if genome in IGENOMES:
|
||||
return genome
|
||||
|
||||
# Check aliases
|
||||
genome_lower = genome.lower()
|
||||
for gid, info in IGENOMES.items():
|
||||
if genome_lower in [a.lower() for a in info.get('aliases', [])]:
|
||||
return gid
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def is_genome_installed(genome_id: str) -> bool:
|
||||
"""Check if genome is installed locally."""
|
||||
cache_dir = get_cache_dir()
|
||||
genome_dir = cache_dir / genome_id
|
||||
|
||||
# Check for fasta as minimum requirement
|
||||
fasta_path = genome_dir / 'genome.fa'
|
||||
return fasta_path.exists()
|
||||
|
||||
|
||||
def get_genome_path(genome_id: str) -> Optional[Path]:
|
||||
"""Get local path to genome if installed."""
|
||||
if not is_genome_installed(genome_id):
|
||||
return None
|
||||
return get_cache_dir() / genome_id
|
||||
|
||||
|
||||
def list_genomes(installed_only: bool = False) -> List[Dict]:
|
||||
"""List available genomes."""
|
||||
result = []
|
||||
|
||||
for genome_id, info in IGENOMES.items():
|
||||
installed = is_genome_installed(genome_id)
|
||||
|
||||
if installed_only and not installed:
|
||||
continue
|
||||
|
||||
genome_path = get_genome_path(genome_id) if installed else None
|
||||
|
||||
result.append({
|
||||
'id': genome_id,
|
||||
'display_name': info['display_name'],
|
||||
'species': info['species'],
|
||||
'aliases': info.get('aliases', []),
|
||||
'installed': installed,
|
||||
'path': str(genome_path) if genome_path else None,
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def download_genome(
|
||||
genome_id: str,
|
||||
components: Optional[List[str]] = None,
|
||||
force: bool = False
|
||||
) -> bool:
|
||||
"""
|
||||
Download genome reference files from iGenomes.
|
||||
|
||||
Args:
|
||||
genome_id: Genome identifier (e.g., GRCh38)
|
||||
components: Specific components to download (fasta, gtf, etc.)
|
||||
force: Overwrite existing files
|
||||
|
||||
Returns:
|
||||
True if successful
|
||||
"""
|
||||
# Resolve genome ID
|
||||
resolved = resolve_genome_id(genome_id)
|
||||
if not resolved:
|
||||
print(f"Unknown genome: {genome_id}")
|
||||
print(f"Available: {', '.join(IGENOMES.keys())}")
|
||||
return False
|
||||
|
||||
genome_id = resolved
|
||||
info = IGENOMES[genome_id]
|
||||
|
||||
# Check for AWS CLI
|
||||
aws_available = subprocess.run(
|
||||
['which', 'aws'],
|
||||
capture_output=True
|
||||
).returncode == 0
|
||||
|
||||
if not aws_available:
|
||||
print("AWS CLI not found. Required for iGenomes download.")
|
||||
print("Install with: pip install awscli")
|
||||
print("\nAlternative: Use --genome flag with nf-core pipelines")
|
||||
print("which will auto-download references (slower, per-run).")
|
||||
return False
|
||||
|
||||
# Create cache directory
|
||||
cache_dir = get_cache_dir()
|
||||
genome_dir = cache_dir / genome_id
|
||||
genome_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Determine components to download
|
||||
if components is None:
|
||||
components = ['fasta', 'gtf'] # Minimum required
|
||||
|
||||
print(f"Downloading {info['display_name']} to {genome_dir}")
|
||||
print(f"Components: {', '.join(components)}")
|
||||
|
||||
success = True
|
||||
for component in components:
|
||||
if component not in info.get('files', {}):
|
||||
print(f" Skipping {component}: not available for {genome_id}")
|
||||
continue
|
||||
|
||||
remote_path = info['files'][component]
|
||||
s3_path = f"{info['s3_base']}/{remote_path}"
|
||||
|
||||
# Determine local path
|
||||
if remote_path.endswith('/'):
|
||||
# Directory (e.g., index)
|
||||
local_path = genome_dir / component
|
||||
else:
|
||||
# File
|
||||
filename = Path(remote_path).name
|
||||
local_path = genome_dir / filename
|
||||
|
||||
if local_path.exists() and not force:
|
||||
print(f" {component}: Already exists (use --force to overwrite)")
|
||||
continue
|
||||
|
||||
print(f" Downloading {component}...")
|
||||
|
||||
# Build AWS command
|
||||
cmd = ['aws', 's3', 'cp', '--no-sign-request']
|
||||
|
||||
if remote_path.endswith('/'):
|
||||
cmd.extend(['--recursive', s3_path, str(local_path)])
|
||||
else:
|
||||
cmd.extend([s3_path, str(local_path)])
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" ERROR downloading {component}:")
|
||||
print(f" {result.stderr[:200]}")
|
||||
success = False
|
||||
else:
|
||||
print(f" {component}: Downloaded successfully")
|
||||
|
||||
if success:
|
||||
print(f"\nGenome {genome_id} ready at: {genome_dir}")
|
||||
else:
|
||||
print(f"\nSome components failed to download.")
|
||||
|
||||
return success
|
||||
|
||||
|
||||
def get_nextflow_params(genome_id: str) -> Dict[str, str]:
|
||||
"""
|
||||
Get Nextflow parameters for a genome.
|
||||
|
||||
Returns dict with --fasta, --gtf if local,
|
||||
or just --genome if using iGenomes key.
|
||||
"""
|
||||
resolved = resolve_genome_id(genome_id)
|
||||
if not resolved:
|
||||
return {'error': f'Unknown genome: {genome_id}'}
|
||||
|
||||
genome_id = resolved
|
||||
|
||||
# Check if installed locally
|
||||
genome_path = get_genome_path(genome_id)
|
||||
|
||||
if genome_path:
|
||||
params = {}
|
||||
|
||||
# Check for local files
|
||||
fasta = genome_path / 'genome.fa'
|
||||
if fasta.exists():
|
||||
params['fasta'] = str(fasta)
|
||||
|
||||
gtf = genome_path / 'genes.gtf'
|
||||
if gtf.exists():
|
||||
params['gtf'] = str(gtf)
|
||||
|
||||
if params:
|
||||
return params
|
||||
|
||||
# Fall back to iGenomes key
|
||||
return {'genome': genome_id}
|
||||
|
||||
|
||||
def print_genome_list(genomes: List[Dict], output_json: bool = False):
|
||||
"""Print genome list."""
|
||||
if output_json:
|
||||
print(json.dumps(genomes, indent=2))
|
||||
return
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print(" Available Genomes")
|
||||
print("=" * 50 + "\n")
|
||||
|
||||
for g in genomes:
|
||||
status = "\033[92m[installed]\033[0m" if g['installed'] else ""
|
||||
print(f" {g['id']}: {g['display_name']} {status}")
|
||||
print(f" Species: {g['species']}")
|
||||
print(f" Aliases: {', '.join(g['aliases'])}")
|
||||
if g['path']:
|
||||
print(f" Path: {g['path']}")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Manage genome references for nf-core pipelines',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Commands:
|
||||
list List available genomes
|
||||
check <genome> Check if genome is installed
|
||||
download <genome> Download genome from iGenomes
|
||||
params <genome> Get Nextflow parameters for genome
|
||||
|
||||
Examples:
|
||||
%(prog)s list
|
||||
%(prog)s list --installed
|
||||
%(prog)s check GRCh38
|
||||
%(prog)s download GRCh38
|
||||
%(prog)s download GRCh38 --components fasta gtf star_index
|
||||
%(prog)s params GRCh38
|
||||
"""
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command', help='Commands')
|
||||
|
||||
# List command
|
||||
list_parser = subparsers.add_parser('list', help='List available genomes')
|
||||
list_parser.add_argument('--installed', action='store_true',
|
||||
help='Show only installed genomes')
|
||||
list_parser.add_argument('--json', action='store_true',
|
||||
help='Output as JSON')
|
||||
|
||||
# Check command
|
||||
check_parser = subparsers.add_parser('check', help='Check if genome is installed')
|
||||
check_parser.add_argument('genome', help='Genome ID (e.g., GRCh38)')
|
||||
check_parser.add_argument('--json', action='store_true',
|
||||
help='Output as JSON')
|
||||
|
||||
# Download command
|
||||
dl_parser = subparsers.add_parser('download', help='Download genome from iGenomes')
|
||||
dl_parser.add_argument('genome', help='Genome ID (e.g., GRCh38)')
|
||||
dl_parser.add_argument('--components', nargs='+',
|
||||
help='Specific components (fasta, gtf, bwa_index, star_index)')
|
||||
dl_parser.add_argument('--force', action='store_true',
|
||||
help='Overwrite existing files')
|
||||
|
||||
# Params command
|
||||
params_parser = subparsers.add_parser('params', help='Get Nextflow params for genome')
|
||||
params_parser.add_argument('genome', help='Genome ID')
|
||||
params_parser.add_argument('--json', action='store_true',
|
||||
help='Output as JSON')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command == 'list':
|
||||
genomes = list_genomes(installed_only=args.installed)
|
||||
print_genome_list(genomes, args.json)
|
||||
|
||||
elif args.command == 'check':
|
||||
resolved = resolve_genome_id(args.genome)
|
||||
if not resolved:
|
||||
print(f"Unknown genome: {args.genome}")
|
||||
sys.exit(1)
|
||||
|
||||
installed = is_genome_installed(resolved)
|
||||
path = get_genome_path(resolved) if installed else None
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({
|
||||
'genome': resolved,
|
||||
'installed': installed,
|
||||
'path': str(path) if path else None
|
||||
}))
|
||||
else:
|
||||
if installed:
|
||||
print(f"✓ Genome {resolved} is installed at: {path}")
|
||||
else:
|
||||
print(f"✗ Genome {resolved} is not installed locally")
|
||||
print(f" Download with: python {sys.argv[0]} download {resolved}")
|
||||
|
||||
sys.exit(0 if installed else 1)
|
||||
|
||||
elif args.command == 'download':
|
||||
success = download_genome(args.genome, args.components, args.force)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.command == 'params':
|
||||
params = get_nextflow_params(args.genome)
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(params))
|
||||
else:
|
||||
if 'error' in params:
|
||||
print(f"Error: {params['error']}")
|
||||
sys.exit(1)
|
||||
|
||||
for key, value in params.items():
|
||||
print(f"--{key} {value}")
|
||||
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,732 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
GEO/SRA Data Fetcher
|
||||
====================
|
||||
Download raw sequencing data from NCBI GEO/SRA and prepare for nf-core pipelines.
|
||||
|
||||
Usage:
|
||||
python sra_geo_fetch.py info <GEO_ID> # Get study information
|
||||
python sra_geo_fetch.py list <GEO_ID> # List all samples/runs
|
||||
python sra_geo_fetch.py download <GEO_ID> -o DIR # Download FASTQ files
|
||||
python sra_geo_fetch.py samplesheet <GEO_ID> ... # Generate samplesheet
|
||||
|
||||
Examples:
|
||||
python sra_geo_fetch.py info GSE110004
|
||||
python sra_geo_fetch.py list GSE110004 --filter "RNA-Seq:PAIRED"
|
||||
python sra_geo_fetch.py download GSE110004 -o ./fastq --parallel 4
|
||||
python sra_geo_fetch.py samplesheet GSE110004 --fastq-dir ./fastq -o samplesheet.csv
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# Add utils to path
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from utils.ncbi_utils import (
|
||||
check_network_access,
|
||||
fetch_geo_metadata,
|
||||
fetch_sra_study_accession,
|
||||
fetch_sra_run_info,
|
||||
fetch_sra_run_info_detailed,
|
||||
fetch_ena_fastq_urls,
|
||||
download_file,
|
||||
format_file_size,
|
||||
estimate_download_size,
|
||||
group_samples_by_type,
|
||||
format_sample_groups_table,
|
||||
)
|
||||
|
||||
# Set up logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Load genome mapping
|
||||
SCRIPT_DIR = Path(__file__).parent
|
||||
GENOMES_FILE = SCRIPT_DIR / "config" / "genomes.yaml"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StudyInfo:
|
||||
"""Information about a GEO study."""
|
||||
geo_id: str
|
||||
title: str
|
||||
organism: str
|
||||
n_samples: int
|
||||
summary: str
|
||||
sra_study: Optional[str]
|
||||
suggested_genome: Optional[str]
|
||||
suggested_pipeline: Optional[str]
|
||||
|
||||
|
||||
def load_genome_mapping() -> Dict:
|
||||
"""Load organism to genome mapping from config."""
|
||||
if not GENOMES_FILE.exists():
|
||||
return {}
|
||||
|
||||
try:
|
||||
import yaml
|
||||
with open(GENOMES_FILE) as f:
|
||||
config = yaml.safe_load(f)
|
||||
return config.get('organisms', {})
|
||||
except ImportError:
|
||||
# Fallback: parse YAML manually for simple cases
|
||||
mapping = {}
|
||||
try:
|
||||
with open(GENOMES_FILE) as f:
|
||||
content = f.read()
|
||||
# Simple regex parsing for organism blocks
|
||||
pattern = r'"([^"]+)":\s*\n\s*genome:\s*"([^"]+)"'
|
||||
for match in re.finditer(pattern, content):
|
||||
mapping[match.group(1)] = {'genome': match.group(2)}
|
||||
except Exception:
|
||||
pass
|
||||
return mapping
|
||||
|
||||
|
||||
def suggest_genome(organism: str) -> Optional[str]:
|
||||
"""Suggest a genome based on organism name."""
|
||||
genome_map = load_genome_mapping()
|
||||
|
||||
# Direct match
|
||||
if organism in genome_map:
|
||||
return genome_map[organism].get('genome')
|
||||
|
||||
# Case-insensitive search
|
||||
organism_lower = organism.lower()
|
||||
for org_name, info in genome_map.items():
|
||||
if org_name.lower() == organism_lower:
|
||||
return info.get('genome')
|
||||
# Check aliases
|
||||
aliases = info.get('aliases', [])
|
||||
if any(alias.lower() == organism_lower for alias in aliases):
|
||||
return info.get('genome')
|
||||
|
||||
# Common fallbacks
|
||||
fallbacks = {
|
||||
'homo sapiens': 'GRCh38',
|
||||
'human': 'GRCh38',
|
||||
'mus musculus': 'GRCm39',
|
||||
'mouse': 'GRCm39',
|
||||
'saccharomyces cerevisiae': 'R64-1-1',
|
||||
'yeast': 'R64-1-1',
|
||||
'drosophila melanogaster': 'BDGP6',
|
||||
'caenorhabditis elegans': 'WBcel235',
|
||||
'danio rerio': 'GRCz11',
|
||||
'arabidopsis thaliana': 'TAIR10',
|
||||
'rattus norvegicus': 'Rnor_6.0',
|
||||
}
|
||||
|
||||
return fallbacks.get(organism_lower)
|
||||
|
||||
|
||||
def suggest_pipeline(library_strategy: str, library_source: str = '') -> str:
|
||||
"""Suggest nf-core pipeline based on library strategy."""
|
||||
strategy = library_strategy.upper()
|
||||
|
||||
pipeline_map = {
|
||||
'RNA-SEQ': 'rnaseq',
|
||||
'ATAC-SEQ': 'atacseq',
|
||||
'CHIP-SEQ': 'chipseq',
|
||||
'WGS': 'sarek',
|
||||
'WXS': 'sarek',
|
||||
'AMPLICON': 'ampliseq',
|
||||
'BISULFITE-SEQ': 'methylseq',
|
||||
'HI-C': 'hic',
|
||||
}
|
||||
|
||||
return pipeline_map.get(strategy, 'rnaseq')
|
||||
|
||||
|
||||
def cmd_info(args):
|
||||
"""Display study information."""
|
||||
geo_id = args.geo_id.upper()
|
||||
|
||||
print(f"\nFetching information for {geo_id}...")
|
||||
|
||||
# Check network
|
||||
network_ok, network_msg = check_network_access()
|
||||
if not network_ok:
|
||||
print(f"\n⚠️ Network issues detected:\n{network_msg}")
|
||||
|
||||
# Get GEO metadata
|
||||
metadata = fetch_geo_metadata(geo_id)
|
||||
if not metadata:
|
||||
print(f"\n❌ Could not fetch metadata for {geo_id}")
|
||||
return 1
|
||||
|
||||
# Get SRA study accession
|
||||
sra_study = fetch_sra_study_accession(geo_id)
|
||||
|
||||
# Get detailed run info
|
||||
print("Fetching SRA run information...")
|
||||
runs = fetch_sra_run_info_detailed(geo_id)
|
||||
if not runs:
|
||||
# Fallback to basic method
|
||||
runs = fetch_sra_run_info(geo_id)
|
||||
|
||||
# Group samples by type
|
||||
groups = group_samples_by_type(runs) if runs else {}
|
||||
|
||||
# Suggest genome and pipeline
|
||||
organism = metadata.get('organism', 'Unknown')
|
||||
genome = suggest_genome(organism)
|
||||
|
||||
# Determine primary data type
|
||||
primary_strategy = 'RNA-SEQ'
|
||||
if groups:
|
||||
primary_group = max(groups.items(), key=lambda x: x[1]['count'])
|
||||
primary_strategy = primary_group[1]['strategy']
|
||||
pipeline = suggest_pipeline(primary_strategy)
|
||||
|
||||
# Estimate download size
|
||||
est_size = estimate_download_size(runs)
|
||||
|
||||
# Display info
|
||||
print("\n" + "━" * 70)
|
||||
print(f"{geo_id}: {metadata.get('title', 'N/A')}")
|
||||
print("━" * 70)
|
||||
print(f"Organism: {organism}")
|
||||
print(f"Samples: {metadata.get('n_samples', 'N/A')}")
|
||||
print(f"SRA Study: {sra_study or 'Not found'}")
|
||||
print(f"Runs: {len(runs)}")
|
||||
print(f"Est. Size: ~{format_file_size(est_size)}")
|
||||
print(f"Genome: {genome or 'Unknown (manual selection required)'}")
|
||||
print(f"Pipeline: nf-core/{pipeline} (suggested)")
|
||||
|
||||
# Show sample groups table
|
||||
if groups:
|
||||
print(format_sample_groups_table(groups))
|
||||
|
||||
if metadata.get('summary'):
|
||||
summary = metadata['summary']
|
||||
if len(summary) > 300:
|
||||
summary = summary[:297] + "..."
|
||||
print(f"\nSummary:\n {summary}")
|
||||
|
||||
print("━" * 70)
|
||||
|
||||
# Show download hints
|
||||
if len(groups) > 1:
|
||||
print("\n💡 To download a specific subset, use:")
|
||||
for key in sorted(groups.keys()):
|
||||
print(f" --subset \"{key}\"")
|
||||
|
||||
# Save study info JSON
|
||||
if args.output_json:
|
||||
info = {
|
||||
'geo_id': geo_id,
|
||||
'title': metadata.get('title'),
|
||||
'organism': organism,
|
||||
'n_samples': metadata.get('n_samples'),
|
||||
'sra_study': sra_study,
|
||||
'n_runs': len(runs),
|
||||
'groups': {k: {**v, 'runs': None, 'gsm_ids': list(v.get('gsm_ids', []))} for k, v in groups.items()},
|
||||
'suggested_genome': genome,
|
||||
'suggested_pipeline': pipeline,
|
||||
'summary': metadata.get('summary'),
|
||||
}
|
||||
output_path = Path(args.output_json)
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(info, f, indent=2)
|
||||
print(f"\n📄 Study info saved to: {output_path}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_groups(args):
|
||||
"""Display sample groups in a study for interactive selection."""
|
||||
geo_id = args.geo_id.upper()
|
||||
|
||||
print(f"\nFetching sample groups for {geo_id}...")
|
||||
|
||||
# Get detailed run info
|
||||
runs = fetch_sra_run_info_detailed(geo_id)
|
||||
if not runs:
|
||||
runs = fetch_sra_run_info(geo_id)
|
||||
|
||||
if not runs:
|
||||
print(f"\n❌ No runs found for {geo_id}")
|
||||
return 1
|
||||
|
||||
# Group samples
|
||||
groups = group_samples_by_type(runs)
|
||||
|
||||
print(format_sample_groups_table(groups))
|
||||
|
||||
# Output for interactive selection
|
||||
print("\n📋 Available groups for --subset option:")
|
||||
for i, (key, info) in enumerate(sorted(groups.items(), key=lambda x: -x[1]['count']), 1):
|
||||
size_str = format_file_size(info['size_estimate'])
|
||||
print(f" {i}. \"{key}\" - {info['count']} samples (~{size_str})")
|
||||
|
||||
# Save to JSON if requested
|
||||
if args.output:
|
||||
output_path = Path(args.output)
|
||||
output_data = {
|
||||
'geo_id': geo_id,
|
||||
'groups': {}
|
||||
}
|
||||
for key, info in groups.items():
|
||||
output_data['groups'][key] = {
|
||||
'count': info['count'],
|
||||
'gsm_range': info['gsm_range'],
|
||||
'gsm_ids': info.get('gsm_ids', []),
|
||||
'size_estimate': info['size_estimate'],
|
||||
'strategy': info['strategy'],
|
||||
'layout': info['layout'],
|
||||
'srr_ids': [r['srr'] for r in info['runs']],
|
||||
}
|
||||
with open(output_path, 'w') as f:
|
||||
json.dump(output_data, f, indent=2)
|
||||
print(f"\n📄 Groups saved to: {output_path}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_list(args):
|
||||
"""List all samples and runs in a study."""
|
||||
geo_id = args.geo_id.upper()
|
||||
|
||||
print(f"\nFetching run list for {geo_id}...")
|
||||
|
||||
runs = fetch_sra_run_info(geo_id)
|
||||
if not runs:
|
||||
print(f"\n❌ No runs found for {geo_id}")
|
||||
return 1
|
||||
|
||||
# Apply filter if specified
|
||||
if args.filter:
|
||||
filter_parts = args.filter.split(':')
|
||||
strategy_filter = filter_parts[0].upper() if filter_parts else None
|
||||
layout_filter = filter_parts[1].upper() if len(filter_parts) > 1 else None
|
||||
|
||||
filtered = []
|
||||
for run in runs:
|
||||
if strategy_filter and run.get('library_strategy', '').upper() != strategy_filter:
|
||||
continue
|
||||
if layout_filter and run.get('layout', '').upper() != layout_filter:
|
||||
continue
|
||||
filtered.append(run)
|
||||
runs = filtered
|
||||
|
||||
print(f"\n{'SRR':<15} {'GSM':<12} {'Layout':<8} {'Strategy':<12} {'Size':>10}")
|
||||
print("-" * 60)
|
||||
|
||||
for run in runs:
|
||||
size = format_file_size(run.get('bases', 0) // 4)
|
||||
print(f"{run['srr']:<15} {run.get('gsm', 'N/A'):<12} {run.get('layout', 'N/A'):<8} "
|
||||
f"{run.get('library_strategy', 'N/A'):<12} {size:>10}")
|
||||
|
||||
print(f"\nTotal: {len(runs)} runs")
|
||||
|
||||
# Output as TSV if requested
|
||||
if args.output:
|
||||
output_path = Path(args.output)
|
||||
with open(output_path, 'w') as f:
|
||||
f.write("run_accession\tgsm\tlayout\tlibrary_strategy\tbases\n")
|
||||
for run in runs:
|
||||
f.write(f"{run['srr']}\t{run.get('gsm', '')}\t{run.get('layout', '')}\t"
|
||||
f"{run.get('library_strategy', '')}\t{run.get('bases', 0)}\n")
|
||||
print(f"\n📄 Run list saved to: {output_path}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def download_fastq_file(url: str, output_path: Path, timeout: int = 600) -> Tuple[str, bool]:
|
||||
"""Download a single FASTQ file."""
|
||||
filename = output_path.name
|
||||
if output_path.exists():
|
||||
return filename, True # Already exists
|
||||
|
||||
success = download_file(url, output_path, timeout=timeout, show_progress=False)
|
||||
return filename, success
|
||||
|
||||
|
||||
def interactive_select_group(groups: Dict[str, Dict]) -> Optional[str]:
|
||||
"""Interactively select a sample group."""
|
||||
if len(groups) <= 1:
|
||||
return None # No selection needed
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(" SELECT SAMPLE GROUP TO DOWNLOAD")
|
||||
print("=" * 60)
|
||||
|
||||
sorted_groups = sorted(groups.items(), key=lambda x: -x[1]['count'])
|
||||
|
||||
for i, (key, info) in enumerate(sorted_groups, 1):
|
||||
size_str = format_file_size(info['size_estimate'])
|
||||
print(f"\n [{i}] {info['strategy']} ({info['layout'].lower()})")
|
||||
print(f" Samples: {info['count']}")
|
||||
print(f" GSM: {info['gsm_range']}")
|
||||
print(f" Size: ~{size_str}")
|
||||
|
||||
print(f"\n [0] Download ALL ({sum(g['count'] for g in groups.values())} samples)")
|
||||
print("-" * 60)
|
||||
|
||||
try:
|
||||
choice = input("\nEnter selection (0-{}): ".format(len(sorted_groups))).strip()
|
||||
choice_num = int(choice)
|
||||
|
||||
if choice_num == 0:
|
||||
return None # Download all
|
||||
elif 1 <= choice_num <= len(sorted_groups):
|
||||
selected_key = sorted_groups[choice_num - 1][0]
|
||||
print(f"\n✓ Selected: {selected_key}")
|
||||
return selected_key
|
||||
else:
|
||||
print("Invalid selection, downloading all.")
|
||||
return None
|
||||
except (ValueError, EOFError, KeyboardInterrupt):
|
||||
print("\nInvalid input, downloading all.")
|
||||
return None
|
||||
|
||||
|
||||
def cmd_download(args):
|
||||
"""Download FASTQ files from ENA."""
|
||||
geo_id = args.geo_id.upper()
|
||||
output_dir = Path(args.output)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"\nPreparing download for {geo_id}...")
|
||||
|
||||
# Get detailed run info (includes BioProject fallback for SuperSeries)
|
||||
print("Fetching SRA run information...")
|
||||
runs = fetch_sra_run_info_detailed(geo_id)
|
||||
if not runs:
|
||||
runs = fetch_sra_run_info(geo_id)
|
||||
|
||||
if not runs:
|
||||
print(f"❌ No runs found for {geo_id}")
|
||||
return 1
|
||||
|
||||
# Collect all unique SRA studies from runs (SuperSeries may have multiple)
|
||||
sra_studies = set(r.get('sra_study', '') for r in runs if r.get('sra_study'))
|
||||
if not sra_studies:
|
||||
print(f"❌ Could not find any SRA studies for {geo_id}")
|
||||
return 1
|
||||
|
||||
if len(sra_studies) > 1:
|
||||
print(f"SuperSeries detected with {len(sra_studies)} SRA studies: {', '.join(sorted(sra_studies))}")
|
||||
else:
|
||||
print(f"SRA Study: {list(sra_studies)[0]}")
|
||||
|
||||
# Group samples
|
||||
groups = group_samples_by_type(runs)
|
||||
|
||||
# Show sample groups if multiple types exist
|
||||
if len(groups) > 1:
|
||||
print(format_sample_groups_table(groups))
|
||||
|
||||
# Handle subset selection
|
||||
selected_subset = args.subset
|
||||
|
||||
# Interactive mode if multiple groups and no subset specified
|
||||
if args.interactive and len(groups) > 1 and not selected_subset:
|
||||
selected_subset = interactive_select_group(groups)
|
||||
|
||||
# Get ENA FASTQ URLs from all SRA studies
|
||||
print("\nFetching FASTQ URLs from ENA...")
|
||||
fastq_urls = {}
|
||||
for sra_study in sorted(sra_studies):
|
||||
study_urls = fetch_ena_fastq_urls(sra_study)
|
||||
if study_urls:
|
||||
print(f" {sra_study}: {len(study_urls)} runs")
|
||||
fastq_urls.update(study_urls)
|
||||
|
||||
if not fastq_urls:
|
||||
print("❌ No FASTQ URLs found in ENA")
|
||||
print("Tip: Try using SRA toolkit directly with prefetch + fasterq-dump")
|
||||
return 1
|
||||
|
||||
# Apply filter if specified
|
||||
if selected_subset:
|
||||
filter_parts = selected_subset.split(':')
|
||||
strategy_filter = filter_parts[0].upper() if filter_parts else None
|
||||
layout_filter = filter_parts[1].upper() if len(filter_parts) > 1 else None
|
||||
|
||||
filtered_srrs = set()
|
||||
for run in runs:
|
||||
if strategy_filter and run.get('library_strategy', '').upper() != strategy_filter:
|
||||
continue
|
||||
if layout_filter and run.get('layout', '').upper() != layout_filter:
|
||||
continue
|
||||
filtered_srrs.add(run['srr'])
|
||||
|
||||
fastq_urls = {srr: urls for srr, urls in fastq_urls.items() if srr in filtered_srrs}
|
||||
print(f"\n📦 Filtered to {len(fastq_urls)} runs matching \"{selected_subset}\"")
|
||||
|
||||
# Count files to download
|
||||
total_files = sum(len(urls) for urls in fastq_urls.values())
|
||||
print(f"\n📦 Found {len(fastq_urls)} runs, {total_files} FASTQ files to download")
|
||||
|
||||
# Check for existing files
|
||||
existing = 0
|
||||
downloads_needed = []
|
||||
for srr, urls in fastq_urls.items():
|
||||
for url in urls:
|
||||
filename = url.split('/')[-1]
|
||||
filepath = output_dir / filename
|
||||
if filepath.exists():
|
||||
existing += 1
|
||||
else:
|
||||
downloads_needed.append((url, filepath))
|
||||
|
||||
if existing:
|
||||
print(f" ✓ {existing} files already exist, skipping")
|
||||
|
||||
if not downloads_needed:
|
||||
print("\n✅ All files already downloaded!")
|
||||
return 0
|
||||
|
||||
print(f" ↓ {len(downloads_needed)} files to download")
|
||||
print()
|
||||
|
||||
# Download files
|
||||
successful = 0
|
||||
failed = []
|
||||
|
||||
if args.parallel > 1:
|
||||
# Parallel download
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as executor:
|
||||
futures = {
|
||||
executor.submit(download_fastq_file, url, filepath): filepath
|
||||
for url, filepath in downloads_needed
|
||||
}
|
||||
|
||||
for i, future in enumerate(as_completed(futures), 1):
|
||||
filepath = futures[future]
|
||||
filename, success = future.result()
|
||||
status = "✓" if success else "✗"
|
||||
print(f" [{i}/{len(downloads_needed)}] {status} {filename}")
|
||||
if success:
|
||||
successful += 1
|
||||
else:
|
||||
failed.append(filename)
|
||||
else:
|
||||
# Sequential download
|
||||
for i, (url, filepath) in enumerate(downloads_needed, 1):
|
||||
filename = filepath.name
|
||||
print(f" [{i}/{len(downloads_needed)}] Downloading {filename}...")
|
||||
success = download_file(url, filepath, timeout=args.timeout)
|
||||
if success:
|
||||
successful += 1
|
||||
print(f" ✓ Done")
|
||||
else:
|
||||
failed.append(filename)
|
||||
print(f" ✗ Failed")
|
||||
|
||||
print(f"\n📊 Download summary:")
|
||||
print(f" ✓ Successful: {successful + existing}")
|
||||
print(f" ✗ Failed: {len(failed)}")
|
||||
|
||||
if failed:
|
||||
print(f"\nFailed downloads:")
|
||||
for f in failed:
|
||||
print(f" - {f}")
|
||||
return 1
|
||||
|
||||
print(f"\n✅ All files downloaded to: {output_dir}")
|
||||
|
||||
# Save metadata
|
||||
metadata_path = output_dir / "download_metadata.json"
|
||||
metadata = {
|
||||
'geo_id': geo_id,
|
||||
'sra_studies': sorted(sra_studies),
|
||||
'n_runs': len(fastq_urls),
|
||||
'n_files': total_files,
|
||||
'output_dir': str(output_dir.absolute()),
|
||||
}
|
||||
with open(metadata_path, 'w') as f:
|
||||
json.dump(metadata, f, indent=2)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_samplesheet(args):
|
||||
"""Generate samplesheet for nf-core pipeline."""
|
||||
geo_id = args.geo_id.upper()
|
||||
fastq_dir = Path(args.fastq_dir)
|
||||
output_path = Path(args.output)
|
||||
|
||||
print(f"\nGenerating samplesheet for {geo_id}...")
|
||||
|
||||
# Get run info
|
||||
runs = fetch_sra_run_info(geo_id)
|
||||
if not runs:
|
||||
print(f"❌ No runs found for {geo_id}")
|
||||
return 1
|
||||
|
||||
# Get GEO metadata for sample naming
|
||||
metadata = fetch_geo_metadata(geo_id)
|
||||
organism = metadata.get('organism', 'Unknown') if metadata else 'Unknown'
|
||||
genome = suggest_genome(organism)
|
||||
|
||||
# Detect pipeline from data
|
||||
strategies = set(r.get('library_strategy', 'RNA-SEQ') for r in runs)
|
||||
primary_strategy = list(strategies)[0] if strategies else 'RNA-SEQ'
|
||||
pipeline = args.pipeline or suggest_pipeline(primary_strategy)
|
||||
|
||||
# Map SRR to local FASTQ files
|
||||
samples = []
|
||||
for run in runs:
|
||||
srr = run['srr']
|
||||
layout = run.get('layout', 'PAIRED')
|
||||
|
||||
# Find FASTQ files
|
||||
if layout == 'PAIRED':
|
||||
r1 = fastq_dir / f"{srr}_1.fastq.gz"
|
||||
r2 = fastq_dir / f"{srr}_2.fastq.gz"
|
||||
if not r1.exists() or not r2.exists():
|
||||
logger.warning(f"FASTQ files not found for {srr}")
|
||||
continue
|
||||
samples.append({
|
||||
'srr': srr,
|
||||
'gsm': run.get('gsm', ''),
|
||||
'fastq_1': str(r1.absolute()),
|
||||
'fastq_2': str(r2.absolute()),
|
||||
'layout': 'PAIRED',
|
||||
})
|
||||
else:
|
||||
r1 = fastq_dir / f"{srr}.fastq.gz"
|
||||
if not r1.exists():
|
||||
r1 = fastq_dir / f"{srr}_1.fastq.gz"
|
||||
if not r1.exists():
|
||||
logger.warning(f"FASTQ file not found for {srr}")
|
||||
continue
|
||||
samples.append({
|
||||
'srr': srr,
|
||||
'gsm': run.get('gsm', ''),
|
||||
'fastq_1': str(r1.absolute()),
|
||||
'fastq_2': '',
|
||||
'layout': 'SINGLE',
|
||||
})
|
||||
|
||||
if not samples:
|
||||
print(f"❌ No FASTQ files found in {fastq_dir}")
|
||||
return 1
|
||||
|
||||
# Generate sample names
|
||||
# Try to infer meaningful names from GSM IDs or use SRR
|
||||
sample_names = {}
|
||||
for sample in samples:
|
||||
# Default to SRR accession
|
||||
sample_names[sample['srr']] = sample['srr']
|
||||
|
||||
# Write samplesheet
|
||||
with open(output_path, 'w') as f:
|
||||
if pipeline == 'rnaseq':
|
||||
f.write("sample,fastq_1,fastq_2,strandedness\n")
|
||||
for sample in samples:
|
||||
name = sample_names[sample['srr']]
|
||||
f.write(f"{name},{sample['fastq_1']},{sample['fastq_2']},auto\n")
|
||||
elif pipeline == 'atacseq':
|
||||
f.write("sample,fastq_1,fastq_2,replicate\n")
|
||||
for i, sample in enumerate(samples, 1):
|
||||
name = sample_names[sample['srr']]
|
||||
f.write(f"{name},{sample['fastq_1']},{sample['fastq_2']},1\n")
|
||||
else:
|
||||
# Generic format
|
||||
f.write("sample,fastq_1,fastq_2\n")
|
||||
for sample in samples:
|
||||
name = sample_names[sample['srr']]
|
||||
f.write(f"{name},{sample['fastq_1']},{sample['fastq_2']}\n")
|
||||
|
||||
print(f"\n✅ Generated samplesheet: {output_path}")
|
||||
print(f" Samples: {len(samples)}")
|
||||
print(f" Pipeline: nf-core/{pipeline}")
|
||||
if genome:
|
||||
print(f" Genome: {genome}")
|
||||
|
||||
print(f"\n💡 Suggested command:")
|
||||
print(f" nextflow run nf-core/{pipeline} \\")
|
||||
print(f" --input {output_path} \\")
|
||||
print(f" --outdir results \\")
|
||||
if genome:
|
||||
print(f" --genome {genome} \\")
|
||||
print(f" -profile docker")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Download GEO/SRA data and prepare for nf-core pipelines",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s info GSE110004 # Get study info with sample groups
|
||||
%(prog)s groups GSE110004 # Show sample groups for selection
|
||||
%(prog)s list GSE110004 --filter RNA-Seq # List RNA-seq runs
|
||||
%(prog)s download GSE110004 -o ./fastq -i # Download with interactive selection
|
||||
%(prog)s download GSE110004 -o ./fastq --subset "RNA-Seq:PAIRED"
|
||||
%(prog)s samplesheet GSE110004 \\
|
||||
--fastq-dir ./fastq -o samplesheet.csv # Generate samplesheet
|
||||
"""
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command', help='Commands')
|
||||
|
||||
# info command
|
||||
info_parser = subparsers.add_parser('info', help='Display study information with sample groups')
|
||||
info_parser.add_argument('geo_id', help='GEO accession (e.g., GSE110004)')
|
||||
info_parser.add_argument('--output-json', '-o', help='Save info to JSON file')
|
||||
|
||||
# groups command
|
||||
groups_parser = subparsers.add_parser('groups', help='Show sample groups for interactive selection')
|
||||
groups_parser.add_argument('geo_id', help='GEO accession')
|
||||
groups_parser.add_argument('--output', '-o', help='Save groups to JSON file')
|
||||
|
||||
# list command
|
||||
list_parser = subparsers.add_parser('list', help='List samples and runs')
|
||||
list_parser.add_argument('geo_id', help='GEO accession')
|
||||
list_parser.add_argument('--filter', '-f', help='Filter by strategy:layout (e.g., RNA-Seq:PAIRED)')
|
||||
list_parser.add_argument('--output', '-o', help='Save to TSV file')
|
||||
|
||||
# download command
|
||||
dl_parser = subparsers.add_parser('download', help='Download FASTQ files')
|
||||
dl_parser.add_argument('geo_id', help='GEO accession')
|
||||
dl_parser.add_argument('--output', '-o', required=True, help='Output directory')
|
||||
dl_parser.add_argument('--subset', '-s', help='Filter subset (e.g., RNA-Seq:PAIRED)')
|
||||
dl_parser.add_argument('--interactive', '-i', action='store_true',
|
||||
help='Interactively select sample group to download')
|
||||
dl_parser.add_argument('--parallel', '-p', type=int, default=4, help='Parallel downloads')
|
||||
dl_parser.add_argument('--timeout', '-t', type=int, default=600, help='Download timeout (sec)')
|
||||
|
||||
# samplesheet command
|
||||
ss_parser = subparsers.add_parser('samplesheet', help='Generate samplesheet')
|
||||
ss_parser.add_argument('geo_id', help='GEO accession')
|
||||
ss_parser.add_argument('--fastq-dir', '-f', required=True, help='Directory with FASTQ files')
|
||||
ss_parser.add_argument('--output', '-o', default='samplesheet.csv', help='Output samplesheet')
|
||||
ss_parser.add_argument('--pipeline', '-p', help='Target pipeline (auto-detected if not specified)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
commands = {
|
||||
'info': cmd_info,
|
||||
'groups': cmd_groups,
|
||||
'list': cmd_list,
|
||||
'download': cmd_download,
|
||||
'samplesheet': cmd_samplesheet,
|
||||
}
|
||||
|
||||
return commands[args.command](args)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Utility modules for nf-core pipeline deployment.
|
||||
|
||||
Modules:
|
||||
ncbi_utils: NCBI/GEO/SRA data fetching and download utilities
|
||||
file_discovery: Find FASTQ, BAM, and CRAM files
|
||||
sample_inference: Extract sample info, detect tumor/normal
|
||||
validators: Validate samplesheets before writing
|
||||
"""
|
||||
|
||||
# NCBI utilities for GEO/SRA data acquisition
|
||||
from .ncbi_utils import (
|
||||
check_network_access,
|
||||
fetch_geo_metadata,
|
||||
fetch_sra_study_accession,
|
||||
fetch_sra_run_info,
|
||||
fetch_sra_run_info_detailed,
|
||||
fetch_bioproject_from_geo,
|
||||
fetch_ena_fastq_urls,
|
||||
download_file,
|
||||
fetch_pubmed_metadata,
|
||||
format_file_size,
|
||||
estimate_download_size,
|
||||
group_samples_by_type,
|
||||
format_sample_groups_table,
|
||||
)
|
||||
|
||||
# File discovery utilities
|
||||
from .file_discovery import discover_files, FileInfo, count_files_by_type
|
||||
|
||||
# Sample inference utilities
|
||||
from .sample_inference import (
|
||||
extract_sample_info,
|
||||
infer_tumor_normal_status,
|
||||
match_read_pairs,
|
||||
extract_replicate_number
|
||||
)
|
||||
|
||||
# Validation utilities
|
||||
from .validators import validate_samplesheet, ValidationResult
|
||||
|
||||
__all__ = [
|
||||
# ncbi_utils
|
||||
'check_network_access',
|
||||
'fetch_geo_metadata',
|
||||
'fetch_sra_study_accession',
|
||||
'fetch_sra_run_info',
|
||||
'fetch_sra_run_info_detailed',
|
||||
'fetch_bioproject_from_geo',
|
||||
'fetch_ena_fastq_urls',
|
||||
'download_file',
|
||||
'fetch_pubmed_metadata',
|
||||
'format_file_size',
|
||||
'estimate_download_size',
|
||||
'group_samples_by_type',
|
||||
'format_sample_groups_table',
|
||||
# file_discovery
|
||||
'discover_files',
|
||||
'FileInfo',
|
||||
'count_files_by_type',
|
||||
# sample_inference
|
||||
'extract_sample_info',
|
||||
'infer_tumor_normal_status',
|
||||
'match_read_pairs',
|
||||
'extract_replicate_number',
|
||||
# validators
|
||||
'validate_samplesheet',
|
||||
'ValidationResult',
|
||||
]
|
||||
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
File discovery utilities for FASTQ, BAM, and CRAM files.
|
||||
|
||||
This module provides functions to recursively discover sequencing data files
|
||||
in a directory structure.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileInfo:
|
||||
"""Information about a discovered file."""
|
||||
path: str
|
||||
name: str
|
||||
stem: str
|
||||
extension: str
|
||||
size: int
|
||||
file_type: str # fastq, bam, cram
|
||||
|
||||
def __repr__(self):
|
||||
return f"FileInfo({self.name}, type={self.file_type})"
|
||||
|
||||
|
||||
# Supported file extensions by type
|
||||
EXTENSIONS = {
|
||||
"fastq": [".fastq.gz", ".fq.gz", ".fastq", ".fq"],
|
||||
"bam": [".bam"],
|
||||
"cram": [".cram"],
|
||||
}
|
||||
|
||||
# Index file extensions
|
||||
INDEX_EXTENSIONS = {
|
||||
"bam": [".bam.bai", ".bai"],
|
||||
"cram": [".cram.crai", ".crai"],
|
||||
}
|
||||
|
||||
|
||||
def discover_files(
|
||||
directory: str,
|
||||
file_type: str = "fastq",
|
||||
follow_symlinks: bool = True
|
||||
) -> List[FileInfo]:
|
||||
"""
|
||||
Recursively discover files of specified type.
|
||||
|
||||
Args:
|
||||
directory: Root directory to search
|
||||
file_type: One of 'fastq', 'bam', 'cram'
|
||||
follow_symlinks: Whether to follow symbolic links
|
||||
|
||||
Returns:
|
||||
List of FileInfo objects sorted by path
|
||||
"""
|
||||
if file_type not in EXTENSIONS:
|
||||
raise ValueError(f"Unknown file type: {file_type}. Supported: {list(EXTENSIONS.keys())}")
|
||||
|
||||
directory = os.path.abspath(directory)
|
||||
if not os.path.isdir(directory):
|
||||
raise ValueError(f"Not a directory: {directory}")
|
||||
|
||||
extensions = EXTENSIONS[file_type]
|
||||
files = []
|
||||
seen_paths = set() # Avoid duplicates from symlinks
|
||||
|
||||
for root, _, filenames in os.walk(directory, followlinks=follow_symlinks):
|
||||
for filename in filenames:
|
||||
# Check each extension
|
||||
for ext in extensions:
|
||||
if filename.lower().endswith(ext.lower()):
|
||||
full_path = os.path.join(root, filename)
|
||||
|
||||
# Resolve to handle symlinks
|
||||
try:
|
||||
real_path = os.path.realpath(full_path)
|
||||
except OSError:
|
||||
real_path = full_path
|
||||
|
||||
if real_path in seen_paths:
|
||||
continue
|
||||
seen_paths.add(real_path)
|
||||
|
||||
try:
|
||||
size = os.path.getsize(full_path)
|
||||
except OSError:
|
||||
size = 0
|
||||
|
||||
# Extract stem (remove extension)
|
||||
stem = filename
|
||||
for e in extensions:
|
||||
if stem.lower().endswith(e.lower()):
|
||||
stem = stem[:-len(e)]
|
||||
break
|
||||
|
||||
files.append(FileInfo(
|
||||
path=full_path,
|
||||
name=filename,
|
||||
stem=stem,
|
||||
extension=ext,
|
||||
size=size,
|
||||
file_type=file_type
|
||||
))
|
||||
break # Found matching extension, no need to check others
|
||||
|
||||
return sorted(files, key=lambda f: f.path)
|
||||
|
||||
|
||||
def count_files_by_type(directory: str) -> Dict[str, int]:
|
||||
"""
|
||||
Count files by type in directory.
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
Dict mapping file_type to count
|
||||
"""
|
||||
counts = {}
|
||||
for file_type in EXTENSIONS:
|
||||
try:
|
||||
files = discover_files(directory, file_type)
|
||||
counts[file_type] = len(files)
|
||||
except (ValueError, PermissionError):
|
||||
counts[file_type] = 0
|
||||
return counts
|
||||
|
||||
|
||||
def find_index_file(alignment_file: str) -> Optional[str]:
|
||||
"""
|
||||
Find index file for a BAM or CRAM file.
|
||||
|
||||
Args:
|
||||
alignment_file: Path to BAM or CRAM file
|
||||
|
||||
Returns:
|
||||
Path to index file if found, None otherwise
|
||||
"""
|
||||
path = Path(alignment_file)
|
||||
|
||||
# Determine file type
|
||||
if path.suffix.lower() == ".bam":
|
||||
index_exts = INDEX_EXTENSIONS["bam"]
|
||||
elif path.suffix.lower() == ".cram":
|
||||
index_exts = INDEX_EXTENSIONS["cram"]
|
||||
else:
|
||||
return None
|
||||
|
||||
# Try common index file patterns
|
||||
for ext in index_exts:
|
||||
# Pattern: file.bam.bai or file.bai
|
||||
if ext.startswith(".bam") or ext.startswith(".cram"):
|
||||
candidate = Path(str(path) + ext.split(".")[-1])
|
||||
else:
|
||||
candidate = path.with_suffix(ext)
|
||||
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
|
||||
# Also try: file.bam -> file.bam.bai
|
||||
candidate = Path(str(path) + "." + ext.lstrip("."))
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def detect_input_type(directory: str) -> str:
|
||||
"""
|
||||
Auto-detect predominant input file type in directory.
|
||||
|
||||
Prioritizes: FASTQ > BAM > CRAM
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
Detected file type ('fastq', 'bam', or 'cram')
|
||||
"""
|
||||
counts = count_files_by_type(directory)
|
||||
|
||||
# Prioritize by preference
|
||||
for file_type in ["fastq", "bam", "cram"]:
|
||||
if counts.get(file_type, 0) > 0:
|
||||
return file_type
|
||||
|
||||
return "fastq" # Default
|
||||
@@ -0,0 +1,808 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
NCBI Utilities for GEO/SRA Data Access
|
||||
======================================
|
||||
Shared utilities for fetching metadata and downloading data from NCBI services.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError, HTTPError
|
||||
|
||||
# Set up logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# NCBI rate limiting - track last request time
|
||||
_last_ncbi_request_time = 0.0
|
||||
_NCBI_MIN_DELAY = 0.34 # 3 requests per second max without API key
|
||||
|
||||
|
||||
def _rate_limit_ncbi():
|
||||
"""Enforce NCBI rate limit of 3 requests/second."""
|
||||
global _last_ncbi_request_time
|
||||
current_time = time.time()
|
||||
elapsed = current_time - _last_ncbi_request_time
|
||||
if elapsed < _NCBI_MIN_DELAY:
|
||||
time.sleep(_NCBI_MIN_DELAY - elapsed)
|
||||
_last_ncbi_request_time = time.time()
|
||||
|
||||
|
||||
# Try to import requests for better HTTP handling
|
||||
try:
|
||||
import requests
|
||||
HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
HAS_REQUESTS = False
|
||||
logger.debug("requests not installed - using urllib fallback")
|
||||
|
||||
|
||||
def check_network_access() -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if NCBI/ENA servers are accessible.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
test_urls = [
|
||||
("NCBI Entrez", "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi"),
|
||||
("NCBI FTP", "https://ftp.ncbi.nlm.nih.gov/"),
|
||||
("ENA API", "https://www.ebi.ac.uk/ena/portal/api/"),
|
||||
]
|
||||
|
||||
results = []
|
||||
for name, url in test_urls:
|
||||
try:
|
||||
if HAS_REQUESTS:
|
||||
# Use GET instead of HEAD - NCBI Entrez returns 405 for HEAD
|
||||
response = requests.get(url, timeout=10)
|
||||
success = response.status_code < 400
|
||||
else:
|
||||
req = Request(url, headers={'User-Agent': 'geo-sra-skill/1.0'})
|
||||
with urlopen(req, timeout=10) as response:
|
||||
success = True
|
||||
results.append((name, success, None))
|
||||
except Exception as e:
|
||||
results.append((name, False, str(e)))
|
||||
|
||||
all_success = all(r[1] for r in results)
|
||||
|
||||
msg_parts = []
|
||||
for name, success, error in results:
|
||||
status = "✓" if success else "✗"
|
||||
msg_parts.append(f" {status} {name}: {'OK' if success else error or 'Failed'}")
|
||||
|
||||
return all_success, "\n".join(msg_parts)
|
||||
|
||||
|
||||
def fetch_geo_metadata(geo_id: str) -> Optional[Dict]:
|
||||
"""
|
||||
Fetch GEO study metadata using NCBI Entrez E-utilities.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
Dict with study metadata or None if failed
|
||||
"""
|
||||
try:
|
||||
# Use esearch to get GEO UID
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term={geo_id}[Accession]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not id_list:
|
||||
logger.warning(f"No GEO entry found for {geo_id}")
|
||||
return None
|
||||
|
||||
# Use esummary to get metadata
|
||||
uid = id_list[0]
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gds&id={uid}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(uid, {})
|
||||
|
||||
return {
|
||||
'geo_id': geo_id,
|
||||
'title': result.get('title', 'N/A'),
|
||||
'summary': result.get('summary', 'N/A'),
|
||||
'organism': result.get('taxon', 'N/A'),
|
||||
'n_samples': result.get('n_samples', 0),
|
||||
'gpl': result.get('gpl', 'N/A'),
|
||||
'entrytype': result.get('entrytype', 'N/A'),
|
||||
'pubmed_ids': result.get('pubmedids', []),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching GEO metadata for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_study_accession(geo_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the SRA study accession (SRPxxxxxx) for a GEO accession.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
SRA study accession (e.g., 'SRP126328') or None
|
||||
"""
|
||||
try:
|
||||
# Search for SRA study linked to GEO
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not id_list:
|
||||
return None
|
||||
|
||||
# Get summary to extract SRP accession
|
||||
uid = id_list[0]
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=sra&id={uid}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(uid, {})
|
||||
exp_xml = result.get('expxml', '')
|
||||
|
||||
# Extract SRP from the XML
|
||||
srp_match = re.search(r'<Study acc="(SRP\d+)"', exp_xml)
|
||||
if srp_match:
|
||||
return srp_match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching SRA study for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_run_info(geo_id: str, bioproject: Optional[str] = None) -> List[Dict]:
|
||||
"""
|
||||
Fetch SRA run information for all samples in a GEO study.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
bioproject: Optional BioProject accession for fallback search
|
||||
|
||||
Returns:
|
||||
List of dicts with run info (srr, gsm, layout, library_strategy, etc.)
|
||||
"""
|
||||
runs = []
|
||||
|
||||
try:
|
||||
# First get the BioProject accession
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
# If no results, try BioProject fallback
|
||||
if not id_list:
|
||||
if not bioproject:
|
||||
bioproject = fetch_bioproject_from_geo(geo_id)
|
||||
|
||||
if bioproject:
|
||||
logger.info(f"Using BioProject {bioproject} for {geo_id}")
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={bioproject}&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
if not id_list:
|
||||
logger.warning(f"No SRA entries found for {geo_id}")
|
||||
return runs
|
||||
|
||||
# Batch fetch summaries
|
||||
ids_str = ','.join(id_list)
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=sra&id={ids_str}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=60)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=60) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {})
|
||||
|
||||
for uid in id_list:
|
||||
entry = result.get(uid, {})
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
exp_xml = entry.get('expxml', '')
|
||||
runs_xml = entry.get('runs', '')
|
||||
|
||||
# Extract metadata from XML
|
||||
layout_match = re.search(r'<LIBRARY_LAYOUT>\s*<(\w+)', exp_xml)
|
||||
strategy_match = re.search(r'<LIBRARY_STRATEGY>(\w+)', exp_xml)
|
||||
source_match = re.search(r'<LIBRARY_SOURCE>(\w+)', exp_xml)
|
||||
gsm_match = re.search(r'<Sample acc="(GSM\d+)"', exp_xml)
|
||||
srx_match = re.search(r'<Experiment acc="(SRX\d+)"', exp_xml)
|
||||
|
||||
# Extract run accessions
|
||||
srr_matches = re.findall(r'<Run acc="(SRR\d+)"[^>]*total_spots="(\d+)"[^>]*total_bases="(\d+)"', runs_xml)
|
||||
|
||||
for srr, spots, bases in srr_matches:
|
||||
runs.append({
|
||||
'srr': srr,
|
||||
'srx': srx_match.group(1) if srx_match else '',
|
||||
'gsm': gsm_match.group(1) if gsm_match else '',
|
||||
'layout': layout_match.group(1).upper() if layout_match else 'UNKNOWN',
|
||||
'library_strategy': strategy_match.group(1) if strategy_match else 'UNKNOWN',
|
||||
'library_source': source_match.group(1) if source_match else 'UNKNOWN',
|
||||
'spots': int(spots),
|
||||
'bases': int(bases),
|
||||
})
|
||||
|
||||
return runs
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching SRA run info for {geo_id}: {e}")
|
||||
return runs
|
||||
|
||||
|
||||
def fetch_ena_fastq_urls(study_accession: str) -> Dict[str, List[str]]:
|
||||
"""
|
||||
Get FASTQ download URLs from ENA for an SRA study.
|
||||
|
||||
ENA provides faster downloads than SRA with pre-split paired files.
|
||||
|
||||
Args:
|
||||
study_accession: SRA study accession (e.g., 'SRP126328')
|
||||
|
||||
Returns:
|
||||
Dict mapping SRR accession to list of FASTQ URLs
|
||||
"""
|
||||
fastq_urls = {}
|
||||
|
||||
try:
|
||||
# Query ENA API
|
||||
ena_url = f"https://www.ebi.ac.uk/ena/portal/api/filereport?accession={study_accession}&result=read_run&fields=run_accession,sample_alias,fastq_ftp&format=tsv"
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(ena_url, timeout=60)
|
||||
content = response.text
|
||||
else:
|
||||
with urlopen(ena_url, timeout=60) as response:
|
||||
content = response.read().decode()
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
if len(lines) < 2:
|
||||
logger.warning(f"No FASTQ URLs found in ENA for {study_accession}")
|
||||
return fastq_urls
|
||||
|
||||
# Parse TSV
|
||||
header = lines[0].split('\t')
|
||||
run_idx = header.index('run_accession') if 'run_accession' in header else 0
|
||||
ftp_idx = header.index('fastq_ftp') if 'fastq_ftp' in header else 2
|
||||
|
||||
for line in lines[1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
fields = line.split('\t')
|
||||
if len(fields) > max(run_idx, ftp_idx):
|
||||
srr = fields[run_idx]
|
||||
ftp_urls = fields[ftp_idx]
|
||||
if ftp_urls:
|
||||
# URLs are semicolon-separated, convert to HTTP URLs
|
||||
# ENA supports both FTP and HTTP, HTTP is easier with requests
|
||||
urls = [f"http://{url}" for url in ftp_urls.split(';') if url]
|
||||
fastq_urls[srr] = urls
|
||||
|
||||
return fastq_urls
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching ENA URLs for {study_accession}: {e}")
|
||||
return fastq_urls
|
||||
|
||||
|
||||
def download_file(url: str, output_path: Path, timeout: int = 300, show_progress: bool = True) -> bool:
|
||||
"""
|
||||
Download a file with progress indication.
|
||||
|
||||
Args:
|
||||
url: URL to download
|
||||
output_path: Path to save file
|
||||
timeout: Download timeout in seconds
|
||||
show_progress: Show progress bar
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(url, stream=True, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
downloaded = 0
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if show_progress and total_size > 0:
|
||||
pct = (downloaded / total_size) * 100
|
||||
print(f"\r Progress: {pct:.1f}%", end='', flush=True)
|
||||
if show_progress:
|
||||
print() # New line after progress
|
||||
return True
|
||||
else:
|
||||
# Fallback to urllib
|
||||
req = Request(url, headers={'User-Agent': 'geo-sra-skill/1.0'})
|
||||
with urlopen(req, timeout=timeout) as response:
|
||||
with open(output_path, 'wb') as f:
|
||||
shutil.copyfileobj(response, f)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Download error for {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def fetch_pubmed_metadata(pmid: str, max_retries: int = 3) -> Optional[Dict]:
|
||||
"""
|
||||
Fetch paper metadata from PubMed.
|
||||
|
||||
Args:
|
||||
pmid: PubMed ID
|
||||
max_retries: Number of retries on failure
|
||||
|
||||
Returns:
|
||||
Dict with 'authors', 'year', 'journal', 'doi' or None
|
||||
"""
|
||||
url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=json"
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(pmid, {})
|
||||
|
||||
if not result or 'error' in result:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
return None
|
||||
|
||||
# Extract authors
|
||||
authors_list = result.get('authors', [])
|
||||
if not authors_list:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
return None
|
||||
|
||||
author_names = [f"{a.get('name', '')}" for a in authors_list[:3]]
|
||||
authors = ', '.join(author_names)
|
||||
if len(authors_list) > 3:
|
||||
authors += ', et al.'
|
||||
|
||||
# Extract year
|
||||
pubdate = result.get('pubdate', '')
|
||||
year_match = re.search(r'\b(20\d{2})\b', pubdate)
|
||||
year = year_match.group(1) if year_match else "Unknown"
|
||||
|
||||
# Extract journal
|
||||
journal = result.get('source', 'Unknown')
|
||||
|
||||
# Extract DOI
|
||||
doi = ""
|
||||
for aid in result.get('articleids', []):
|
||||
if aid.get('idtype') == 'doi':
|
||||
doi = aid.get('value', '')
|
||||
break
|
||||
|
||||
return {
|
||||
'authors': authors,
|
||||
'year': year,
|
||||
'journal': journal,
|
||||
'doi': doi,
|
||||
'title': result.get('title', '')
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"PubMed fetch error for PMID {pmid} (attempt {attempt + 1}): {e}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""Format file size in human-readable format."""
|
||||
if size_bytes < 1024:
|
||||
return f"{size_bytes} B"
|
||||
elif size_bytes < 1024 * 1024:
|
||||
return f"{size_bytes / 1024:.1f} KB"
|
||||
elif size_bytes < 1024 * 1024 * 1024:
|
||||
return f"{size_bytes / (1024 * 1024):.1f} MB"
|
||||
else:
|
||||
return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB"
|
||||
|
||||
|
||||
def estimate_download_size(runs: List[Dict]) -> int:
|
||||
"""
|
||||
Estimate total download size from SRA run info.
|
||||
|
||||
Args:
|
||||
runs: List of run info dicts with 'bases' field
|
||||
|
||||
Returns:
|
||||
Estimated size in bytes (rough estimate based on bases)
|
||||
"""
|
||||
total_bases = sum(r.get('bases', 0) for r in runs)
|
||||
# FASTQ is roughly 1 byte per base when compressed
|
||||
return total_bases // 4 # Rough compression ratio
|
||||
|
||||
|
||||
def fetch_bioproject_from_geo(geo_id: str) -> Optional[str]:
|
||||
"""
|
||||
Fetch BioProject accession linked to a GEO study.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
BioProject accession (e.g., 'PRJNA432544') or None
|
||||
"""
|
||||
try:
|
||||
# First get GDS UID
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term={geo_id}[Accession]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
gds_ids = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not gds_ids:
|
||||
return None
|
||||
|
||||
# Get linked BioProject
|
||||
elink_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=gds&db=bioproject&id={gds_ids[0]}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(elink_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(elink_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
linksets = data.get('linksets', [])
|
||||
if linksets and linksets[0].get('linksetdbs'):
|
||||
for linksetdb in linksets[0]['linksetdbs']:
|
||||
if linksetdb.get('dbto') == 'bioproject':
|
||||
bp_ids = linksetdb.get('links', [])
|
||||
if bp_ids:
|
||||
# Get BioProject accession
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=bioproject&id={bp_ids[0]}&retmode=json"
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(str(bp_ids[0]), {})
|
||||
return result.get('project_acc')
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching BioProject for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_run_info_detailed(geo_id: str, bioproject: Optional[str] = None) -> List[Dict]:
|
||||
"""
|
||||
Fetch detailed SRA run information using efetch CSV format.
|
||||
|
||||
This provides richer metadata than esummary, including sample names.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
bioproject: Optional BioProject accession for fallback search
|
||||
|
||||
Returns:
|
||||
List of dicts with detailed run info
|
||||
"""
|
||||
runs = []
|
||||
|
||||
try:
|
||||
# First get SRA UIDs using GEO search
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
# If no results with GEO search, try BioProject
|
||||
if not id_list:
|
||||
# Try to find BioProject if not provided
|
||||
if not bioproject:
|
||||
logger.info(f"No direct SRA link for {geo_id}, searching for BioProject...")
|
||||
bioproject = fetch_bioproject_from_geo(geo_id)
|
||||
|
||||
if bioproject:
|
||||
logger.info(f"Found BioProject: {bioproject}")
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={bioproject}&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
if not id_list:
|
||||
logger.warning(f"No SRA entries found for {geo_id}")
|
||||
return runs
|
||||
|
||||
# Fetch run info in CSV format using efetch
|
||||
ids_str = ','.join(id_list)
|
||||
efetch_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=sra&id={ids_str}&rettype=runinfo&retmode=csv"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(efetch_url, timeout=60)
|
||||
content = response.text
|
||||
else:
|
||||
with urlopen(efetch_url, timeout=60) as response:
|
||||
content = response.read().decode()
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
if len(lines) < 1:
|
||||
return runs
|
||||
|
||||
# NCBI efetch runinfo CSV doesn't include headers
|
||||
# Define the fixed column order for SRA runinfo format
|
||||
header = [
|
||||
'Run', 'ReleaseDate', 'LoadDate', 'spots', 'bases', 'spots_with_mates',
|
||||
'avgLength', 'size_MB', 'AssemblyName', 'download_path', 'Experiment',
|
||||
'LibraryName', 'LibraryStrategy', 'LibrarySelection', 'LibrarySource',
|
||||
'LibraryLayout', 'InsertSize', 'InsertDev', 'Platform', 'Model',
|
||||
'SRAStudy', 'BioProject', 'Study_Pubmed_id', 'ProjectID', 'Sample',
|
||||
'BioSample', 'SampleType', 'TaxID', 'ScientificName', 'SampleName',
|
||||
'g1k_pop_code', 'source', 'g1k_analysis_group', 'Subject_ID', 'Sex',
|
||||
'Disease', 'Tumor', 'Affection_Status', 'Analyte_Type', 'Histological_Type',
|
||||
'Body_Site', 'CenterName', 'Submission', 'dbgap_study_accession', 'Consent',
|
||||
'RunHash', 'ReadHash'
|
||||
]
|
||||
|
||||
# Map column names to indices
|
||||
col_map = {col: idx for idx, col in enumerate(header)}
|
||||
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Handle CSV fields (some may contain commas in quotes)
|
||||
fields = _parse_csv_line(line)
|
||||
if len(fields) < len(header):
|
||||
continue
|
||||
|
||||
def get_field(name, default=''):
|
||||
idx = col_map.get(name, -1)
|
||||
return fields[idx] if idx >= 0 and idx < len(fields) else default
|
||||
|
||||
run = {
|
||||
'srr': get_field('Run'),
|
||||
'srx': get_field('Experiment'),
|
||||
'gsm': get_field('SampleName'), # Often GSM ID
|
||||
'sample_name': get_field('SampleName'),
|
||||
'library_name': get_field('LibraryName'),
|
||||
'layout': get_field('LibraryLayout', 'UNKNOWN').upper(),
|
||||
'library_strategy': get_field('LibraryStrategy', 'UNKNOWN'),
|
||||
'library_source': get_field('LibrarySource', 'UNKNOWN'),
|
||||
'library_selection': get_field('LibrarySelection', ''),
|
||||
'platform': get_field('Platform'),
|
||||
'model': get_field('Model'),
|
||||
'organism': get_field('ScientificName', ''),
|
||||
'spots': int(get_field('spots', 0) or 0),
|
||||
'bases': int(get_field('bases', 0) or 0),
|
||||
'size_mb': float(get_field('size_MB', 0) or 0),
|
||||
'bioproject': get_field('BioProject'),
|
||||
'biosample': get_field('BioSample'),
|
||||
'sra_study': get_field('SRAStudy'),
|
||||
}
|
||||
|
||||
# Only add if we have a valid SRR
|
||||
if run['srr'].startswith('SRR'):
|
||||
runs.append(run)
|
||||
|
||||
return runs
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching detailed SRA run info for {geo_id}: {e}")
|
||||
return runs
|
||||
|
||||
|
||||
def _parse_csv_line(line: str) -> List[str]:
|
||||
"""Parse a CSV line handling quoted fields."""
|
||||
import csv
|
||||
import io
|
||||
reader = csv.reader(io.StringIO(line))
|
||||
for row in reader:
|
||||
return row
|
||||
return []
|
||||
|
||||
|
||||
def group_samples_by_type(runs: List[Dict]) -> Dict[str, Dict]:
|
||||
"""
|
||||
Group SRA runs by library type and layout.
|
||||
|
||||
Returns dict with group names as keys and info dicts as values:
|
||||
{
|
||||
'RNA-Seq:PAIRED': {
|
||||
'runs': [...],
|
||||
'count': 18,
|
||||
'gsm_range': 'GSM2879618-GSM2879635',
|
||||
'size_estimate': 50000000000,
|
||||
'description': 'RNA-Seq paired-end'
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
groups = {}
|
||||
|
||||
for run in runs:
|
||||
strategy = run.get('library_strategy', 'UNKNOWN')
|
||||
layout = run.get('layout', 'UNKNOWN')
|
||||
key = f"{strategy}:{layout}"
|
||||
|
||||
if key not in groups:
|
||||
groups[key] = {
|
||||
'runs': [],
|
||||
'gsm_ids': set(),
|
||||
'total_bases': 0,
|
||||
'strategy': strategy,
|
||||
'layout': layout,
|
||||
}
|
||||
|
||||
groups[key]['runs'].append(run)
|
||||
gsm = run.get('gsm', '')
|
||||
if gsm.startswith('GSM'):
|
||||
groups[key]['gsm_ids'].add(gsm)
|
||||
groups[key]['total_bases'] += run.get('bases', 0)
|
||||
|
||||
# Post-process groups
|
||||
result = {}
|
||||
for key, info in groups.items():
|
||||
gsm_list = sorted(info['gsm_ids'])
|
||||
gsm_range = _format_gsm_range(gsm_list) if gsm_list else 'N/A'
|
||||
|
||||
result[key] = {
|
||||
'runs': info['runs'],
|
||||
'count': len(info['runs']),
|
||||
'gsm_range': gsm_range,
|
||||
'gsm_ids': gsm_list,
|
||||
'size_estimate': info['total_bases'] // 4, # Rough compressed size
|
||||
'strategy': info['strategy'],
|
||||
'layout': info['layout'],
|
||||
'description': f"{info['strategy']} {info['layout'].lower()}",
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _format_gsm_range(gsm_list: List[str]) -> str:
|
||||
"""Format list of GSM IDs as a range if consecutive."""
|
||||
if not gsm_list:
|
||||
return 'N/A'
|
||||
|
||||
if len(gsm_list) == 1:
|
||||
return gsm_list[0]
|
||||
|
||||
# Extract numbers and check if consecutive
|
||||
try:
|
||||
numbers = [int(gsm.replace('GSM', '')) for gsm in gsm_list]
|
||||
numbers.sort()
|
||||
|
||||
if numbers[-1] - numbers[0] == len(numbers) - 1:
|
||||
# Consecutive
|
||||
return f"GSM{numbers[0]}-GSM{numbers[-1]}"
|
||||
else:
|
||||
# Not consecutive, show count
|
||||
return f"{gsm_list[0]}...({len(gsm_list)} samples)"
|
||||
except ValueError:
|
||||
return f"{len(gsm_list)} samples"
|
||||
|
||||
|
||||
def format_sample_groups_table(groups: Dict[str, Dict]) -> str:
|
||||
"""Format sample groups as a readable table."""
|
||||
lines = []
|
||||
lines.append("")
|
||||
lines.append(f"{'Sample Group':<20} {'Count':>6} {'Layout':<10} {'GSM Range':<25} {'Est. Size':>12}")
|
||||
lines.append("-" * 80)
|
||||
|
||||
for key, info in sorted(groups.items(), key=lambda x: -x[1]['count']):
|
||||
size_str = format_file_size(info['size_estimate'])
|
||||
lines.append(
|
||||
f"{info['strategy']:<20} {info['count']:>6} {info['layout']:<10} "
|
||||
f"{info['gsm_range']:<25} {size_str:>12}"
|
||||
)
|
||||
|
||||
lines.append("-" * 80)
|
||||
total_runs = sum(g['count'] for g in groups.values())
|
||||
total_size = sum(g['size_estimate'] for g in groups.values())
|
||||
lines.append(f"{'TOTAL':<20} {total_runs:>6} {'':<10} {'':<25} {format_file_size(total_size):>12}")
|
||||
|
||||
return '\n'.join(lines)
|
||||
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Sample name and metadata inference from filenames.
|
||||
|
||||
This module extracts sample information, detects tumor/normal status,
|
||||
and matches R1/R2 read pairs from sequencing file names.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
# R1/R2 patterns with priority scores (higher = more confident)
|
||||
R1_PATTERNS = [
|
||||
(r'_R1_\d{3}', 10), # _R1_001 (Illumina standard)
|
||||
(r'_R1[_.]', 8), # _R1. or _R1_
|
||||
(r'\.R1[_.]', 8), # .R1. or .R1_
|
||||
(r'_1[_.]', 5), # _1. or _1_
|
||||
(r'_R1\.f', 6), # _R1.fastq
|
||||
(r'_1\.f', 4), # _1.fastq
|
||||
]
|
||||
|
||||
R2_PATTERNS = [
|
||||
(r'_R2_\d{3}', 10), # _R2_001 (Illumina standard)
|
||||
(r'_R2[_.]', 8), # _R2. or _R2_
|
||||
(r'\.R2[_.]', 8), # .R2. or .R2_
|
||||
(r'_2[_.]', 5), # _2. or _2_
|
||||
(r'_R2\.f', 6), # _R2.fastq
|
||||
(r'_2\.f', 4), # _2.fastq
|
||||
]
|
||||
|
||||
# Tumor/normal keywords
|
||||
TUMOR_KEYWORDS = [
|
||||
r'\btumou?r\b',
|
||||
r'\bmetastasis\b',
|
||||
r'\bmet\b',
|
||||
r'\bprimary\b',
|
||||
r'\bcancer\b',
|
||||
r'\bmalignant\b',
|
||||
r'[-_]T[-_]',
|
||||
r'[-_]T\d*$',
|
||||
r'^T\d*[-_]',
|
||||
]
|
||||
|
||||
NORMAL_KEYWORDS = [
|
||||
r'\bnormal\b',
|
||||
r'\bgermline\b',
|
||||
r'\bblood\b',
|
||||
r'\bpbmc\b',
|
||||
r'\bcontrol\b',
|
||||
r'\bhealthy\b',
|
||||
r'\bmatched\b',
|
||||
r'[-_]N[-_]',
|
||||
r'[-_]N\d*$',
|
||||
r'^N\d*[-_]',
|
||||
]
|
||||
|
||||
# Lane pattern
|
||||
LANE_PATTERN = r'[_.]L(\d{3})[_.]'
|
||||
|
||||
# Patient/sample extraction patterns
|
||||
PATIENT_PATTERNS = [
|
||||
r'^(P\d+)[-_]', # P001_sample
|
||||
r'^(patient\d+)[-_]', # patient1_sample
|
||||
r'^(TCGA-\w+-\w+)', # TCGA format
|
||||
r'^([A-Z]{2,3}\d{3,})[-_]', # AB123_sample
|
||||
]
|
||||
|
||||
# Replicate patterns
|
||||
REPLICATE_PATTERNS = [
|
||||
r'[_.]rep(\d+)', # _rep1, .rep2
|
||||
r'[_.]replicate(\d+)', # _replicate1
|
||||
r'[_.]R(\d+)[_.]', # _R1_ (but not R1/R2 for reads!)
|
||||
r'[-_](\d+)$', # sample_1 (last resort)
|
||||
]
|
||||
|
||||
|
||||
def extract_sample_info(filepath: str) -> Dict[str, str]:
|
||||
"""
|
||||
Extract sample metadata from filepath.
|
||||
|
||||
Args:
|
||||
filepath: Path to sequencing file
|
||||
|
||||
Returns:
|
||||
Dict with: sample, patient, lane (if detectable)
|
||||
"""
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
# Remove extensions
|
||||
stem = filename
|
||||
for ext in ['.fastq.gz', '.fq.gz', '.fastq', '.fq', '.bam', '.cram', '.bai', '.crai']:
|
||||
if stem.lower().endswith(ext):
|
||||
stem = stem[:-len(ext)]
|
||||
break
|
||||
|
||||
info = {}
|
||||
|
||||
# Extract lane
|
||||
lane_match = re.search(LANE_PATTERN, stem)
|
||||
info['lane'] = f"L{lane_match.group(1)}" if lane_match else "L001"
|
||||
|
||||
# Remove lane from stem
|
||||
clean_stem = re.sub(LANE_PATTERN, '_', stem)
|
||||
|
||||
# Remove R1/R2 indicators and everything after
|
||||
for pattern, _ in R1_PATTERNS + R2_PATTERNS:
|
||||
clean_stem = re.sub(pattern + r'.*', '', clean_stem, flags=re.IGNORECASE)
|
||||
|
||||
# Clean up trailing/multiple underscores and dots
|
||||
clean_stem = re.sub(r'[_.-]+$', '', clean_stem)
|
||||
clean_stem = re.sub(r'[_.-]{2,}', '_', clean_stem)
|
||||
|
||||
# Try to extract patient ID
|
||||
for pattern in PATIENT_PATTERNS:
|
||||
match = re.match(pattern, clean_stem, re.IGNORECASE)
|
||||
if match:
|
||||
info['patient'] = match.group(1)
|
||||
break
|
||||
|
||||
# Sample is the cleaned stem
|
||||
info['sample'] = clean_stem if clean_stem else filename.split('.')[0]
|
||||
|
||||
# Default patient to sample if not extracted
|
||||
if 'patient' not in info:
|
||||
info['patient'] = info['sample']
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def infer_tumor_normal_status(sample_name: str) -> Optional[int]:
|
||||
"""
|
||||
Infer tumor (1) or normal (0) status from sample name.
|
||||
|
||||
Args:
|
||||
sample_name: Sample identifier
|
||||
|
||||
Returns:
|
||||
1 for tumor, 0 for normal, None if cannot determine
|
||||
"""
|
||||
name_lower = sample_name.lower()
|
||||
|
||||
# Check tumor indicators
|
||||
for pattern in TUMOR_KEYWORDS:
|
||||
if re.search(pattern, name_lower, re.IGNORECASE):
|
||||
return 1
|
||||
|
||||
# Check normal indicators
|
||||
for pattern in NORMAL_KEYWORDS:
|
||||
if re.search(pattern, name_lower, re.IGNORECASE):
|
||||
return 0
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_replicate_number(sample_name: str) -> Optional[int]:
|
||||
"""
|
||||
Extract replicate number from sample name.
|
||||
|
||||
Args:
|
||||
sample_name: Sample identifier
|
||||
|
||||
Returns:
|
||||
Replicate number if found, None otherwise
|
||||
"""
|
||||
for pattern in REPLICATE_PATTERNS:
|
||||
match = re.search(pattern, sample_name, re.IGNORECASE)
|
||||
if match:
|
||||
try:
|
||||
return int(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _get_pattern_score(filename: str, patterns: List[Tuple[str, int]]) -> int:
|
||||
"""Get highest matching pattern score."""
|
||||
max_score = 0
|
||||
for pattern, score in patterns:
|
||||
if re.search(pattern, filename, re.IGNORECASE):
|
||||
max_score = max(max_score, score)
|
||||
return max_score
|
||||
|
||||
|
||||
def _get_sample_key(filepath: str) -> str:
|
||||
"""Generate a key for grouping related files."""
|
||||
info = extract_sample_info(filepath)
|
||||
sample = info['sample']
|
||||
lane = info.get('lane', 'L001')
|
||||
|
||||
# Include lane in key for multi-lane samples
|
||||
if lane != "L001":
|
||||
return f"{sample}_{lane}"
|
||||
return sample
|
||||
|
||||
|
||||
def match_read_pairs(files) -> Dict[str, Dict]:
|
||||
"""
|
||||
Match R1/R2 read pairs using scored pattern matching.
|
||||
|
||||
Args:
|
||||
files: List of FileInfo objects (from file_discovery)
|
||||
|
||||
Returns:
|
||||
Dict mapping sample_key to {'r1': path, 'r2': path, 'info': dict}
|
||||
"""
|
||||
# Classify files
|
||||
r1_files = []
|
||||
r2_files = []
|
||||
|
||||
for file in files:
|
||||
filename = file.name if hasattr(file, 'name') else os.path.basename(str(file))
|
||||
filepath = file.path if hasattr(file, 'path') else str(file)
|
||||
|
||||
r1_score = _get_pattern_score(filename, R1_PATTERNS)
|
||||
r2_score = _get_pattern_score(filename, R2_PATTERNS)
|
||||
|
||||
if r2_score > r1_score and r2_score > 0:
|
||||
r2_files.append((filepath, r2_score))
|
||||
elif r1_score > 0:
|
||||
r1_files.append((filepath, r1_score))
|
||||
else:
|
||||
# No clear indicator - assume R1 (single-end or non-standard naming)
|
||||
r1_files.append((filepath, 0))
|
||||
|
||||
# Build pairs by matching sample keys
|
||||
pairs = {}
|
||||
|
||||
# Process R1 files first
|
||||
for r1_path, score in r1_files:
|
||||
key = _get_sample_key(r1_path)
|
||||
info = extract_sample_info(r1_path)
|
||||
|
||||
if key not in pairs:
|
||||
pairs[key] = {
|
||||
'r1': r1_path,
|
||||
'r2': None,
|
||||
'info': info,
|
||||
'score': score
|
||||
}
|
||||
else:
|
||||
# Multiple R1 files for same sample (should not happen)
|
||||
pairs[key]['r1'] = r1_path
|
||||
|
||||
# Match R2 files
|
||||
for r2_path, score in r2_files:
|
||||
key = _get_sample_key(r2_path)
|
||||
info = extract_sample_info(r2_path)
|
||||
|
||||
if key in pairs:
|
||||
pairs[key]['r2'] = r2_path
|
||||
else:
|
||||
# R2 without matching R1
|
||||
pairs[key] = {
|
||||
'r1': None,
|
||||
'r2': r2_path,
|
||||
'info': info,
|
||||
'score': score
|
||||
}
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def infer_patient_groupings(sample_names: List[str]) -> Dict[str, str]:
|
||||
"""
|
||||
Infer patient groupings from sample names.
|
||||
|
||||
Groups samples that share a common prefix pattern.
|
||||
|
||||
Args:
|
||||
sample_names: List of sample identifiers
|
||||
|
||||
Returns:
|
||||
Dict mapping sample_name to patient_id
|
||||
"""
|
||||
patient_map = {}
|
||||
|
||||
for sample in sample_names:
|
||||
# Try to find a patient pattern
|
||||
for pattern in PATIENT_PATTERNS:
|
||||
match = re.match(pattern, sample, re.IGNORECASE)
|
||||
if match:
|
||||
patient_map[sample] = match.group(1)
|
||||
break
|
||||
|
||||
if sample not in patient_map:
|
||||
# Default: each sample is its own patient
|
||||
patient_map[sample] = sample
|
||||
|
||||
return patient_map
|
||||
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
Samplesheet validation utilities.
|
||||
|
||||
Validates samplesheet rows against pipeline configuration before writing,
|
||||
catching errors early with helpful messages.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Result of samplesheet validation."""
|
||||
valid: bool
|
||||
errors: List[str] = field(default_factory=list)
|
||||
warnings: List[str] = field(default_factory=list)
|
||||
suggestions: List[str] = field(default_factory=list)
|
||||
|
||||
def __bool__(self):
|
||||
return self.valid
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Generate human-readable summary."""
|
||||
lines = []
|
||||
if self.errors:
|
||||
lines.append("Errors:")
|
||||
for e in self.errors:
|
||||
lines.append(f" - {e}")
|
||||
if self.warnings:
|
||||
lines.append("Warnings:")
|
||||
for w in self.warnings:
|
||||
lines.append(f" - {w}")
|
||||
if self.suggestions:
|
||||
lines.append("Suggestions:")
|
||||
for s in self.suggestions:
|
||||
lines.append(f" - {s}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def load_pipeline_config(pipeline: str) -> Optional[Dict]:
|
||||
"""Load pipeline configuration from YAML file."""
|
||||
# Find config directory relative to this file
|
||||
script_dir = Path(__file__).parent.parent.parent
|
||||
config_path = script_dir / "config" / "pipelines" / f"{pipeline}.yaml"
|
||||
|
||||
if not config_path.exists():
|
||||
return None
|
||||
|
||||
with open(config_path) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def validate_samplesheet(
|
||||
rows: List[Dict],
|
||||
pipeline: str,
|
||||
config: Optional[Dict] = None
|
||||
) -> ValidationResult:
|
||||
"""
|
||||
Validate samplesheet rows against pipeline requirements.
|
||||
|
||||
Args:
|
||||
rows: List of row dictionaries
|
||||
pipeline: Pipeline name (e.g., 'rnaseq', 'sarek')
|
||||
config: Optional pre-loaded config dict
|
||||
|
||||
Returns:
|
||||
ValidationResult with errors, warnings, and suggestions
|
||||
"""
|
||||
errors = []
|
||||
warnings = []
|
||||
suggestions = []
|
||||
|
||||
# Load config if not provided
|
||||
if config is None:
|
||||
config = load_pipeline_config(pipeline)
|
||||
|
||||
if config is None:
|
||||
errors.append(f"Unknown pipeline: {pipeline}")
|
||||
return ValidationResult(valid=False, errors=errors)
|
||||
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
required_cols = [c["name"] for c in columns if c.get("required", False)]
|
||||
|
||||
if not rows:
|
||||
errors.append("Samplesheet is empty - no samples found")
|
||||
return ValidationResult(valid=False, errors=errors)
|
||||
|
||||
# Validate each row
|
||||
for i, row in enumerate(rows):
|
||||
row_num = i + 2 # Account for header row
|
||||
|
||||
# Check required columns
|
||||
for col_name in required_cols:
|
||||
col_config = next((c for c in columns if c["name"] == col_name), None)
|
||||
|
||||
# Skip columns with conditions that don't apply
|
||||
if col_config and "condition" in col_config:
|
||||
# Simple condition check - skip for now
|
||||
# Full implementation would evaluate conditions
|
||||
pass
|
||||
|
||||
if col_name not in row or row[col_name] is None or row[col_name] == "":
|
||||
# Check if there's a default
|
||||
if col_config and "default" in col_config:
|
||||
continue
|
||||
errors.append(f"Row {row_num}: Missing required column '{col_name}'")
|
||||
|
||||
# Validate path columns exist
|
||||
for col_name in ["fastq_1", "fastq_2", "bam", "bai"]:
|
||||
if col_name in row and row[col_name]:
|
||||
path = row[col_name]
|
||||
if not os.path.exists(path):
|
||||
errors.append(f"Row {row_num}: File not found: {path}")
|
||||
elif not os.path.isfile(path):
|
||||
errors.append(f"Row {row_num}: Not a file: {path}")
|
||||
|
||||
# Validate enum values
|
||||
for col_config in columns:
|
||||
col_name = col_config["name"]
|
||||
if col_name in row and row[col_name] and "allowed" in col_config:
|
||||
value = row[col_name]
|
||||
allowed = col_config["allowed"]
|
||||
if value not in allowed:
|
||||
errors.append(
|
||||
f"Row {row_num}: Invalid value '{value}' for '{col_name}'. "
|
||||
f"Allowed: {allowed}"
|
||||
)
|
||||
|
||||
# Check R1/R2 pairing consistency
|
||||
r1 = row.get("fastq_1", "")
|
||||
r2 = row.get("fastq_2", "")
|
||||
if r1 and not r2:
|
||||
warnings.append(f"Row {row_num}: Single-end data (no R2 file)")
|
||||
elif r2 and not r1:
|
||||
errors.append(f"Row {row_num}: R2 present but R1 missing")
|
||||
|
||||
# Check for duplicate samples
|
||||
sample_col = "sample" if "sample" in rows[0] else "patient"
|
||||
if sample_col in rows[0]:
|
||||
samples = [r.get(sample_col, "") for r in rows]
|
||||
duplicates = [s for s in set(samples) if samples.count(s) > 1]
|
||||
if duplicates:
|
||||
warnings.append(f"Duplicate sample names: {duplicates}")
|
||||
suggestions.append(
|
||||
"Duplicates may be intentional (multi-lane sequencing). "
|
||||
"Verify sample grouping is correct."
|
||||
)
|
||||
|
||||
# Pipeline-specific validation
|
||||
if pipeline == "sarek":
|
||||
_validate_sarek_specific(rows, errors, warnings, suggestions)
|
||||
elif pipeline == "atacseq":
|
||||
_validate_atacseq_specific(rows, errors, warnings, suggestions)
|
||||
|
||||
return ValidationResult(
|
||||
valid=len(errors) == 0,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
suggestions=suggestions
|
||||
)
|
||||
|
||||
|
||||
def _validate_sarek_specific(
|
||||
rows: List[Dict],
|
||||
errors: List[str],
|
||||
warnings: List[str],
|
||||
suggestions: List[str]
|
||||
):
|
||||
"""Sarek-specific validation for tumor/normal pairing."""
|
||||
# Group by patient
|
||||
patients = {}
|
||||
for row in rows:
|
||||
patient = row.get("patient", "")
|
||||
status = row.get("status")
|
||||
|
||||
if patient not in patients:
|
||||
patients[patient] = {"tumor": 0, "normal": 0, "unknown": 0}
|
||||
|
||||
if status == 1:
|
||||
patients[patient]["tumor"] += 1
|
||||
elif status == 0:
|
||||
patients[patient]["normal"] += 1
|
||||
else:
|
||||
patients[patient]["unknown"] += 1
|
||||
|
||||
# Check pairing
|
||||
for patient, counts in patients.items():
|
||||
if counts["tumor"] > 0 and counts["normal"] == 0:
|
||||
warnings.append(
|
||||
f"Patient '{patient}': Tumor sample(s) without matched normal. "
|
||||
"Somatic calling works best with paired tumor-normal."
|
||||
)
|
||||
suggestions.append(
|
||||
f"For patient '{patient}': Add a normal sample or use tumor-only mode."
|
||||
)
|
||||
|
||||
if counts["unknown"] > 0:
|
||||
warnings.append(
|
||||
f"Patient '{patient}': {counts['unknown']} sample(s) with unknown status. "
|
||||
"Set status column to 0 (normal) or 1 (tumor)."
|
||||
)
|
||||
|
||||
|
||||
def _validate_atacseq_specific(
|
||||
rows: List[Dict],
|
||||
errors: List[str],
|
||||
warnings: List[str],
|
||||
suggestions: List[str]
|
||||
):
|
||||
"""ATAC-seq specific validation for replicates."""
|
||||
# Group by sample (condition)
|
||||
samples = {}
|
||||
for row in rows:
|
||||
sample = row.get("sample", "")
|
||||
replicate = row.get("replicate", 1)
|
||||
|
||||
if sample not in samples:
|
||||
samples[sample] = []
|
||||
|
||||
samples[sample].append(replicate)
|
||||
|
||||
# Check replicates
|
||||
for sample, reps in samples.items():
|
||||
if len(reps) < 2:
|
||||
warnings.append(
|
||||
f"Sample '{sample}': Only {len(reps)} replicate(s). "
|
||||
"Consensus peaks require 2+ replicates."
|
||||
)
|
||||
|
||||
# Check for duplicate replicate numbers
|
||||
if len(reps) != len(set(reps)):
|
||||
errors.append(
|
||||
f"Sample '{sample}': Duplicate replicate numbers detected. "
|
||||
"Each replicate must have a unique number."
|
||||
)
|
||||
|
||||
# Check all samples have R2 (ATAC-seq requires paired-end)
|
||||
for i, row in enumerate(rows):
|
||||
if not row.get("fastq_2"):
|
||||
errors.append(
|
||||
f"Row {i+2}: ATAC-seq requires paired-end data. R2 file missing."
|
||||
)
|
||||
|
||||
|
||||
def validate_file_exists(path: str) -> bool:
|
||||
"""Check if file exists and is accessible."""
|
||||
return os.path.isfile(path) and os.access(path, os.R_OK)
|
||||
|
||||
|
||||
def validate_absolute_path(path: str) -> bool:
|
||||
"""Check if path is absolute."""
|
||||
return os.path.isabs(path)
|
||||
Reference in New Issue
Block a user