wehub-resource-sync
456 lines
15 KiB
Python
456 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Enhanced nf-core samplesheet generator.
|
|
|
|
Features:
|
|
- FASTQ, BAM, and CRAM support
|
|
- Tumor/normal status inference for sarek
|
|
- Robust R1/R2 matching with scoring
|
|
- Pre-write validation with clear error messages
|
|
- Pipeline config-driven column generation
|
|
|
|
Usage:
|
|
python generate_samplesheet.py /path/to/data rnaseq -o samplesheet.csv
|
|
python generate_samplesheet.py /path/to/bams sarek --input-type bam
|
|
python generate_samplesheet.py --validate samplesheet.csv rnaseq
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
import yaml
|
|
|
|
# Add parent directory to path for utils import
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
from utils.file_discovery import discover_files, detect_input_type, find_index_file
|
|
from utils.sample_inference import (
|
|
extract_sample_info,
|
|
infer_tumor_normal_status,
|
|
match_read_pairs,
|
|
extract_replicate_number
|
|
)
|
|
from utils.validators import validate_samplesheet, ValidationResult
|
|
|
|
|
|
def load_pipeline_config(pipeline: str) -> Dict:
|
|
"""Load pipeline configuration from YAML."""
|
|
config_dir = Path(__file__).parent / "config" / "pipelines"
|
|
config_file = config_dir / f"{pipeline}.yaml"
|
|
|
|
if not config_file.exists():
|
|
available = [f.stem for f in config_dir.glob("*.yaml") if not f.stem.startswith("_")]
|
|
raise ValueError(f"Unknown pipeline '{pipeline}'. Available: {', '.join(available)}")
|
|
|
|
with open(config_file) as f:
|
|
return yaml.safe_load(f)
|
|
|
|
|
|
def generate_samplesheet(
|
|
input_dir: str,
|
|
pipeline: str,
|
|
output_file: Optional[str] = None,
|
|
input_type: str = "auto",
|
|
single_end: bool = False,
|
|
interactive: bool = True
|
|
) -> Tuple[Optional[str], ValidationResult]:
|
|
"""
|
|
Generate samplesheet for specified pipeline.
|
|
|
|
Args:
|
|
input_dir: Directory containing sequencing files
|
|
pipeline: Pipeline name (rnaseq, sarek, atacseq)
|
|
output_file: Output CSV path (default: samplesheet_{pipeline}.csv)
|
|
input_type: File type (auto, fastq, bam, cram)
|
|
single_end: Suppress pairing warnings for single-end data
|
|
interactive: Prompt for missing info
|
|
|
|
Returns:
|
|
Tuple of (output_path, validation_result)
|
|
"""
|
|
config = load_pipeline_config(pipeline)
|
|
samplesheet_config = config.get("samplesheet", {})
|
|
supported_types = samplesheet_config.get("input_types", ["fastq"])
|
|
|
|
# Determine input type
|
|
if input_type == "auto":
|
|
input_type = detect_input_type(input_dir)
|
|
print(f"Auto-detected input type: {input_type.upper()}")
|
|
|
|
if input_type not in supported_types:
|
|
return None, ValidationResult(
|
|
valid=False,
|
|
errors=[f"Pipeline '{pipeline}' does not support {input_type.upper()} input. "
|
|
f"Supported: {supported_types}"]
|
|
)
|
|
|
|
# Discover files
|
|
try:
|
|
files = discover_files(input_dir, input_type)
|
|
except ValueError as e:
|
|
return None, ValidationResult(valid=False, errors=[str(e)])
|
|
|
|
if not files:
|
|
return None, ValidationResult(
|
|
valid=False,
|
|
errors=[f"No {input_type.upper()} files found in {input_dir}"],
|
|
suggestions=[
|
|
"Check directory path is correct",
|
|
"Verify file extensions (.fastq.gz, .fq.gz, .bam, .cram)",
|
|
f"Run: ls {input_dir}"
|
|
]
|
|
)
|
|
|
|
print(f"Found {len(files)} {input_type.upper()} files")
|
|
|
|
# Process based on input type
|
|
if input_type == "fastq":
|
|
rows = _process_fastq_files(files, config, single_end)
|
|
else:
|
|
rows = _process_alignment_files(files, config, input_type)
|
|
|
|
if not rows:
|
|
return None, ValidationResult(
|
|
valid=False,
|
|
errors=["Could not generate any samplesheet rows from files"]
|
|
)
|
|
|
|
print(f"Generated {len(rows)} samplesheet rows")
|
|
|
|
# Pipeline-specific processing
|
|
if pipeline == "sarek":
|
|
rows = _process_sarek_samples(rows, interactive)
|
|
elif pipeline == "atacseq":
|
|
rows = _process_atacseq_samples(rows)
|
|
|
|
# Validate before writing
|
|
validation = validate_samplesheet(rows, pipeline, config)
|
|
|
|
if not validation.valid:
|
|
print("\nValidation errors:")
|
|
for error in validation.errors:
|
|
print(f" - {error}")
|
|
|
|
if interactive:
|
|
response = input("\nProceed anyway? [y/N]: ").strip().lower()
|
|
if response != 'y':
|
|
return None, validation
|
|
elif validation.warnings:
|
|
print("\nWarnings:")
|
|
for warning in validation.warnings:
|
|
print(f" - {warning}")
|
|
|
|
# Determine output path
|
|
output_path = output_file or f"samplesheet_{pipeline}.csv"
|
|
|
|
# Write samplesheet
|
|
_write_samplesheet(rows, config, output_path)
|
|
|
|
print(f"\nGenerated: {output_path}")
|
|
print(f" Pipeline: {pipeline} v{config.get('version', 'unknown')}")
|
|
print(f" Samples: {len(set(r.get('sample', r.get('patient', '')) for r in rows))}")
|
|
print(f" Rows: {len(rows)}")
|
|
|
|
# Preview
|
|
_print_preview(rows, config)
|
|
|
|
return output_path, validation
|
|
|
|
|
|
def _process_fastq_files(files, config: Dict, single_end: bool) -> List[Dict]:
|
|
"""Process FASTQ files into samplesheet rows."""
|
|
pairs = match_read_pairs(files)
|
|
|
|
if not pairs:
|
|
return []
|
|
|
|
# Check for unpaired files
|
|
unpaired = [k for k, v in pairs.items() if v.get('r1') and not v.get('r2')]
|
|
if unpaired and not single_end:
|
|
print(f"\nNote: {len(unpaired)} samples appear to be single-end (no R2)")
|
|
|
|
rows = []
|
|
columns = config.get("samplesheet", {}).get("columns", [])
|
|
|
|
for sample_key, pair_info in sorted(pairs.items()):
|
|
if not pair_info.get('r1'):
|
|
continue # Skip entries with only R2
|
|
|
|
info = pair_info.get('info', {})
|
|
|
|
row = {
|
|
'sample': info.get('sample', sample_key),
|
|
'fastq_1': str(Path(pair_info['r1']).absolute()),
|
|
'fastq_2': str(Path(pair_info['r2']).absolute()) if pair_info.get('r2') else '',
|
|
}
|
|
|
|
# Add additional info from filename
|
|
if 'patient' in [c['name'] for c in columns]:
|
|
row['patient'] = info.get('patient', info.get('sample', sample_key))
|
|
|
|
if 'lane' in [c['name'] for c in columns]:
|
|
row['lane'] = info.get('lane', 'L001')
|
|
|
|
# Apply defaults from config
|
|
for col in columns:
|
|
if col['name'] not in row and 'default' in col:
|
|
row[col['name']] = col['default']
|
|
|
|
rows.append(row)
|
|
|
|
return rows
|
|
|
|
|
|
def _process_alignment_files(files, config: Dict, input_type: str) -> List[Dict]:
|
|
"""Process BAM/CRAM files into samplesheet rows."""
|
|
rows = []
|
|
columns = config.get("samplesheet", {}).get("columns", [])
|
|
|
|
for file_info in files:
|
|
# Find index file
|
|
index_path = find_index_file(file_info.path)
|
|
|
|
info = extract_sample_info(file_info.path)
|
|
|
|
row = {
|
|
'sample': info.get('sample', file_info.stem),
|
|
'bam': str(Path(file_info.path).absolute()),
|
|
'bai': str(Path(index_path).absolute()) if index_path else '',
|
|
}
|
|
|
|
# Add patient for sarek
|
|
if 'patient' in [c['name'] for c in columns]:
|
|
row['patient'] = info.get('patient', info.get('sample', file_info.stem))
|
|
|
|
# Apply defaults
|
|
for col in columns:
|
|
if col['name'] not in row and 'default' in col:
|
|
row[col['name']] = col['default']
|
|
|
|
# Warn if no index found
|
|
if not index_path:
|
|
print(f" Warning: No index found for {file_info.name}")
|
|
|
|
rows.append(row)
|
|
|
|
return rows
|
|
|
|
|
|
def _process_sarek_samples(rows: List[Dict], interactive: bool) -> List[Dict]:
|
|
"""Process sarek samples: infer and confirm tumor/normal status."""
|
|
# Auto-infer status from sample names
|
|
for row in rows:
|
|
sample_name = row.get('sample', '')
|
|
inferred = infer_tumor_normal_status(sample_name)
|
|
if inferred is not None:
|
|
row['status'] = inferred
|
|
|
|
# Report inference results
|
|
inferred_tumor = [r for r in rows if r.get('status') == 1]
|
|
inferred_normal = [r for r in rows if r.get('status') == 0]
|
|
unknown = [r for r in rows if 'status' not in r]
|
|
|
|
if inferred_tumor or inferred_normal:
|
|
print(f"\nTumor/normal inference:")
|
|
print(f" Tumor samples: {len(inferred_tumor)}")
|
|
print(f" Normal samples: {len(inferred_normal)}")
|
|
|
|
# Handle unknown samples
|
|
if unknown and interactive:
|
|
print(f"\n{len(unknown)} sample(s) with unknown status:")
|
|
for r in unknown:
|
|
print(f" - {r.get('sample')}")
|
|
|
|
print("\nSpecify status for each (0=normal, 1=tumor, Enter=skip):")
|
|
for r in unknown:
|
|
response = input(f" {r.get('sample')} [0/1/Enter]: ").strip()
|
|
if response in ['0', '1']:
|
|
r['status'] = int(response)
|
|
else:
|
|
r['status'] = 0 # Default to normal
|
|
print(f" Defaulting to normal (0)")
|
|
elif unknown:
|
|
# Non-interactive: default to normal
|
|
for r in unknown:
|
|
r['status'] = 0
|
|
|
|
return rows
|
|
|
|
|
|
def _process_atacseq_samples(rows: List[Dict]) -> List[Dict]:
|
|
"""Process ATAC-seq samples: ensure replicate numbers."""
|
|
# Group by sample name
|
|
sample_counts = {}
|
|
for row in rows:
|
|
sample = row.get('sample', '')
|
|
if sample not in sample_counts:
|
|
sample_counts[sample] = 0
|
|
sample_counts[sample] += 1
|
|
|
|
# Assign replicate numbers if not present
|
|
sample_rep = {}
|
|
for row in rows:
|
|
sample = row.get('sample', '')
|
|
|
|
if 'replicate' not in row or not row['replicate']:
|
|
# Try to extract from filename
|
|
extracted = extract_replicate_number(row.get('fastq_1', ''))
|
|
if extracted:
|
|
row['replicate'] = extracted
|
|
else:
|
|
# Auto-assign sequential
|
|
if sample not in sample_rep:
|
|
sample_rep[sample] = 0
|
|
sample_rep[sample] += 1
|
|
row['replicate'] = sample_rep[sample]
|
|
|
|
return rows
|
|
|
|
|
|
def _write_samplesheet(rows: List[Dict], config: Dict, output_path: str):
|
|
"""Write samplesheet to CSV file."""
|
|
columns = config.get("samplesheet", {}).get("columns", [])
|
|
column_names = [c['name'] for c in columns]
|
|
|
|
# Filter to columns that have data
|
|
active_columns = [c for c in column_names if any(c in row and row[c] for row in rows)]
|
|
|
|
# Ensure fastq_1/fastq_2 or bam/bai are included
|
|
for required in ['fastq_1', 'bam']:
|
|
if required in column_names and required not in active_columns:
|
|
if any(required in row for row in rows):
|
|
active_columns.append(required)
|
|
|
|
# Maintain original column order
|
|
active_columns = [c for c in column_names if c in active_columns]
|
|
|
|
with open(output_path, 'w') as f:
|
|
f.write(','.join(active_columns) + '\n')
|
|
for row in rows:
|
|
values = [str(row.get(col, '')) for col in active_columns]
|
|
f.write(','.join(values) + '\n')
|
|
|
|
|
|
def _print_preview(rows: List[Dict], config: Dict):
|
|
"""Print preview of generated samplesheet."""
|
|
columns = config.get("samplesheet", {}).get("columns", [])
|
|
column_names = [c['name'] for c in columns]
|
|
active_columns = [c for c in column_names if any(c in row for row in rows)]
|
|
|
|
print(f"\nPreview (first 3 rows):")
|
|
print(','.join(active_columns))
|
|
for row in rows[:3]:
|
|
values = [str(row.get(col, ''))[:40] for col in active_columns] # Truncate long paths
|
|
print(','.join(values))
|
|
if len(rows) > 3:
|
|
print(f"... ({len(rows) - 3} more rows)")
|
|
|
|
|
|
def validate_existing_samplesheet(csv_path: str, pipeline: str) -> ValidationResult:
|
|
"""Validate an existing samplesheet file."""
|
|
import csv
|
|
|
|
if not os.path.exists(csv_path):
|
|
return ValidationResult(valid=False, errors=[f"File not found: {csv_path}"])
|
|
|
|
try:
|
|
with open(csv_path, 'r') as f:
|
|
reader = csv.DictReader(f)
|
|
rows = list(reader)
|
|
except Exception as e:
|
|
return ValidationResult(valid=False, errors=[f"Failed to read CSV: {e}"])
|
|
|
|
if not rows:
|
|
return ValidationResult(valid=False, errors=["Samplesheet is empty"])
|
|
|
|
config = load_pipeline_config(pipeline)
|
|
return validate_samplesheet(rows, pipeline, config)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description='Generate nf-core samplesheet from data directory',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Generate samplesheet for RNA-seq
|
|
%(prog)s ./fastqs rnaseq -o samples.csv
|
|
|
|
# Generate samplesheet for sarek from BAM files
|
|
%(prog)s ./bams sarek --input-type bam
|
|
|
|
# Validate existing samplesheet
|
|
%(prog)s --validate samplesheet.csv rnaseq
|
|
|
|
Supported pipelines: rnaseq, sarek, atacseq
|
|
"""
|
|
)
|
|
|
|
parser.add_argument('input', help='Directory with data files, or CSV path for --validate')
|
|
parser.add_argument('pipeline', help='Pipeline name (rnaseq, sarek, atacseq)')
|
|
parser.add_argument('-o', '--output', help='Output CSV filename')
|
|
parser.add_argument('--input-type', choices=['auto', 'fastq', 'bam', 'cram'],
|
|
default='auto', help='Input file type (default: auto-detect)')
|
|
parser.add_argument('--single-end', action='store_true',
|
|
help='Treat as single-end data (suppress pairing warnings)')
|
|
parser.add_argument('--validate', action='store_true',
|
|
help='Validate existing samplesheet instead of generating')
|
|
parser.add_argument('--no-interactive', action='store_true',
|
|
help='Non-interactive mode (use defaults)')
|
|
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
if args.validate:
|
|
# Validate existing samplesheet
|
|
result = validate_existing_samplesheet(args.input, args.pipeline)
|
|
if result.valid:
|
|
print(f"✓ Samplesheet is valid for {args.pipeline}")
|
|
if result.warnings:
|
|
print("\nWarnings:")
|
|
for w in result.warnings:
|
|
print(f" - {w}")
|
|
sys.exit(0)
|
|
else:
|
|
print(f"✗ Samplesheet validation failed")
|
|
print(result.summary())
|
|
sys.exit(1)
|
|
else:
|
|
# Generate new samplesheet
|
|
if not os.path.isdir(args.input):
|
|
print(f"Error: Not a directory: {args.input}")
|
|
sys.exit(1)
|
|
|
|
output_path, result = generate_samplesheet(
|
|
args.input,
|
|
args.pipeline,
|
|
args.output,
|
|
args.input_type,
|
|
args.single_end,
|
|
interactive=not args.no_interactive
|
|
)
|
|
|
|
if output_path is None:
|
|
print("\nFailed to generate samplesheet.")
|
|
if result.suggestions:
|
|
print("\nSuggestions:")
|
|
for s in result.suggestions:
|
|
print(f" - {s}")
|
|
sys.exit(1)
|
|
|
|
sys.exit(0)
|
|
|
|
except ValueError as e:
|
|
print(f"Error: {e}")
|
|
sys.exit(1)
|
|
except KeyboardInterrupt:
|
|
print("\nAborted.")
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|