chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Utility modules for nf-core pipeline deployment.
|
||||
|
||||
Modules:
|
||||
ncbi_utils: NCBI/GEO/SRA data fetching and download utilities
|
||||
file_discovery: Find FASTQ, BAM, and CRAM files
|
||||
sample_inference: Extract sample info, detect tumor/normal
|
||||
validators: Validate samplesheets before writing
|
||||
"""
|
||||
|
||||
# NCBI utilities for GEO/SRA data acquisition
|
||||
from .ncbi_utils import (
|
||||
check_network_access,
|
||||
fetch_geo_metadata,
|
||||
fetch_sra_study_accession,
|
||||
fetch_sra_run_info,
|
||||
fetch_sra_run_info_detailed,
|
||||
fetch_bioproject_from_geo,
|
||||
fetch_ena_fastq_urls,
|
||||
download_file,
|
||||
fetch_pubmed_metadata,
|
||||
format_file_size,
|
||||
estimate_download_size,
|
||||
group_samples_by_type,
|
||||
format_sample_groups_table,
|
||||
)
|
||||
|
||||
# File discovery utilities
|
||||
from .file_discovery import discover_files, FileInfo, count_files_by_type
|
||||
|
||||
# Sample inference utilities
|
||||
from .sample_inference import (
|
||||
extract_sample_info,
|
||||
infer_tumor_normal_status,
|
||||
match_read_pairs,
|
||||
extract_replicate_number
|
||||
)
|
||||
|
||||
# Validation utilities
|
||||
from .validators import validate_samplesheet, ValidationResult
|
||||
|
||||
__all__ = [
|
||||
# ncbi_utils
|
||||
'check_network_access',
|
||||
'fetch_geo_metadata',
|
||||
'fetch_sra_study_accession',
|
||||
'fetch_sra_run_info',
|
||||
'fetch_sra_run_info_detailed',
|
||||
'fetch_bioproject_from_geo',
|
||||
'fetch_ena_fastq_urls',
|
||||
'download_file',
|
||||
'fetch_pubmed_metadata',
|
||||
'format_file_size',
|
||||
'estimate_download_size',
|
||||
'group_samples_by_type',
|
||||
'format_sample_groups_table',
|
||||
# file_discovery
|
||||
'discover_files',
|
||||
'FileInfo',
|
||||
'count_files_by_type',
|
||||
# sample_inference
|
||||
'extract_sample_info',
|
||||
'infer_tumor_normal_status',
|
||||
'match_read_pairs',
|
||||
'extract_replicate_number',
|
||||
# validators
|
||||
'validate_samplesheet',
|
||||
'ValidationResult',
|
||||
]
|
||||
@@ -0,0 +1,189 @@
|
||||
"""
|
||||
File discovery utilities for FASTQ, BAM, and CRAM files.
|
||||
|
||||
This module provides functions to recursively discover sequencing data files
|
||||
in a directory structure.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileInfo:
|
||||
"""Information about a discovered file."""
|
||||
path: str
|
||||
name: str
|
||||
stem: str
|
||||
extension: str
|
||||
size: int
|
||||
file_type: str # fastq, bam, cram
|
||||
|
||||
def __repr__(self):
|
||||
return f"FileInfo({self.name}, type={self.file_type})"
|
||||
|
||||
|
||||
# Supported file extensions by type
|
||||
EXTENSIONS = {
|
||||
"fastq": [".fastq.gz", ".fq.gz", ".fastq", ".fq"],
|
||||
"bam": [".bam"],
|
||||
"cram": [".cram"],
|
||||
}
|
||||
|
||||
# Index file extensions
|
||||
INDEX_EXTENSIONS = {
|
||||
"bam": [".bam.bai", ".bai"],
|
||||
"cram": [".cram.crai", ".crai"],
|
||||
}
|
||||
|
||||
|
||||
def discover_files(
|
||||
directory: str,
|
||||
file_type: str = "fastq",
|
||||
follow_symlinks: bool = True
|
||||
) -> List[FileInfo]:
|
||||
"""
|
||||
Recursively discover files of specified type.
|
||||
|
||||
Args:
|
||||
directory: Root directory to search
|
||||
file_type: One of 'fastq', 'bam', 'cram'
|
||||
follow_symlinks: Whether to follow symbolic links
|
||||
|
||||
Returns:
|
||||
List of FileInfo objects sorted by path
|
||||
"""
|
||||
if file_type not in EXTENSIONS:
|
||||
raise ValueError(f"Unknown file type: {file_type}. Supported: {list(EXTENSIONS.keys())}")
|
||||
|
||||
directory = os.path.abspath(directory)
|
||||
if not os.path.isdir(directory):
|
||||
raise ValueError(f"Not a directory: {directory}")
|
||||
|
||||
extensions = EXTENSIONS[file_type]
|
||||
files = []
|
||||
seen_paths = set() # Avoid duplicates from symlinks
|
||||
|
||||
for root, _, filenames in os.walk(directory, followlinks=follow_symlinks):
|
||||
for filename in filenames:
|
||||
# Check each extension
|
||||
for ext in extensions:
|
||||
if filename.lower().endswith(ext.lower()):
|
||||
full_path = os.path.join(root, filename)
|
||||
|
||||
# Resolve to handle symlinks
|
||||
try:
|
||||
real_path = os.path.realpath(full_path)
|
||||
except OSError:
|
||||
real_path = full_path
|
||||
|
||||
if real_path in seen_paths:
|
||||
continue
|
||||
seen_paths.add(real_path)
|
||||
|
||||
try:
|
||||
size = os.path.getsize(full_path)
|
||||
except OSError:
|
||||
size = 0
|
||||
|
||||
# Extract stem (remove extension)
|
||||
stem = filename
|
||||
for e in extensions:
|
||||
if stem.lower().endswith(e.lower()):
|
||||
stem = stem[:-len(e)]
|
||||
break
|
||||
|
||||
files.append(FileInfo(
|
||||
path=full_path,
|
||||
name=filename,
|
||||
stem=stem,
|
||||
extension=ext,
|
||||
size=size,
|
||||
file_type=file_type
|
||||
))
|
||||
break # Found matching extension, no need to check others
|
||||
|
||||
return sorted(files, key=lambda f: f.path)
|
||||
|
||||
|
||||
def count_files_by_type(directory: str) -> Dict[str, int]:
|
||||
"""
|
||||
Count files by type in directory.
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
Dict mapping file_type to count
|
||||
"""
|
||||
counts = {}
|
||||
for file_type in EXTENSIONS:
|
||||
try:
|
||||
files = discover_files(directory, file_type)
|
||||
counts[file_type] = len(files)
|
||||
except (ValueError, PermissionError):
|
||||
counts[file_type] = 0
|
||||
return counts
|
||||
|
||||
|
||||
def find_index_file(alignment_file: str) -> Optional[str]:
|
||||
"""
|
||||
Find index file for a BAM or CRAM file.
|
||||
|
||||
Args:
|
||||
alignment_file: Path to BAM or CRAM file
|
||||
|
||||
Returns:
|
||||
Path to index file if found, None otherwise
|
||||
"""
|
||||
path = Path(alignment_file)
|
||||
|
||||
# Determine file type
|
||||
if path.suffix.lower() == ".bam":
|
||||
index_exts = INDEX_EXTENSIONS["bam"]
|
||||
elif path.suffix.lower() == ".cram":
|
||||
index_exts = INDEX_EXTENSIONS["cram"]
|
||||
else:
|
||||
return None
|
||||
|
||||
# Try common index file patterns
|
||||
for ext in index_exts:
|
||||
# Pattern: file.bam.bai or file.bai
|
||||
if ext.startswith(".bam") or ext.startswith(".cram"):
|
||||
candidate = Path(str(path) + ext.split(".")[-1])
|
||||
else:
|
||||
candidate = path.with_suffix(ext)
|
||||
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
|
||||
# Also try: file.bam -> file.bam.bai
|
||||
candidate = Path(str(path) + "." + ext.lstrip("."))
|
||||
if candidate.exists():
|
||||
return str(candidate)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def detect_input_type(directory: str) -> str:
|
||||
"""
|
||||
Auto-detect predominant input file type in directory.
|
||||
|
||||
Prioritizes: FASTQ > BAM > CRAM
|
||||
|
||||
Args:
|
||||
directory: Directory to scan
|
||||
|
||||
Returns:
|
||||
Detected file type ('fastq', 'bam', or 'cram')
|
||||
"""
|
||||
counts = count_files_by_type(directory)
|
||||
|
||||
# Prioritize by preference
|
||||
for file_type in ["fastq", "bam", "cram"]:
|
||||
if counts.get(file_type, 0) > 0:
|
||||
return file_type
|
||||
|
||||
return "fastq" # Default
|
||||
@@ -0,0 +1,808 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
NCBI Utilities for GEO/SRA Data Access
|
||||
======================================
|
||||
Shared utilities for fetching metadata and downloading data from NCBI services.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError, HTTPError
|
||||
|
||||
# Set up logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# NCBI rate limiting - track last request time
|
||||
_last_ncbi_request_time = 0.0
|
||||
_NCBI_MIN_DELAY = 0.34 # 3 requests per second max without API key
|
||||
|
||||
|
||||
def _rate_limit_ncbi():
|
||||
"""Enforce NCBI rate limit of 3 requests/second."""
|
||||
global _last_ncbi_request_time
|
||||
current_time = time.time()
|
||||
elapsed = current_time - _last_ncbi_request_time
|
||||
if elapsed < _NCBI_MIN_DELAY:
|
||||
time.sleep(_NCBI_MIN_DELAY - elapsed)
|
||||
_last_ncbi_request_time = time.time()
|
||||
|
||||
|
||||
# Try to import requests for better HTTP handling
|
||||
try:
|
||||
import requests
|
||||
HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
HAS_REQUESTS = False
|
||||
logger.debug("requests not installed - using urllib fallback")
|
||||
|
||||
|
||||
def check_network_access() -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if NCBI/ENA servers are accessible.
|
||||
|
||||
Returns:
|
||||
Tuple of (success, message)
|
||||
"""
|
||||
test_urls = [
|
||||
("NCBI Entrez", "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/einfo.fcgi"),
|
||||
("NCBI FTP", "https://ftp.ncbi.nlm.nih.gov/"),
|
||||
("ENA API", "https://www.ebi.ac.uk/ena/portal/api/"),
|
||||
]
|
||||
|
||||
results = []
|
||||
for name, url in test_urls:
|
||||
try:
|
||||
if HAS_REQUESTS:
|
||||
# Use GET instead of HEAD - NCBI Entrez returns 405 for HEAD
|
||||
response = requests.get(url, timeout=10)
|
||||
success = response.status_code < 400
|
||||
else:
|
||||
req = Request(url, headers={'User-Agent': 'geo-sra-skill/1.0'})
|
||||
with urlopen(req, timeout=10) as response:
|
||||
success = True
|
||||
results.append((name, success, None))
|
||||
except Exception as e:
|
||||
results.append((name, False, str(e)))
|
||||
|
||||
all_success = all(r[1] for r in results)
|
||||
|
||||
msg_parts = []
|
||||
for name, success, error in results:
|
||||
status = "✓" if success else "✗"
|
||||
msg_parts.append(f" {status} {name}: {'OK' if success else error or 'Failed'}")
|
||||
|
||||
return all_success, "\n".join(msg_parts)
|
||||
|
||||
|
||||
def fetch_geo_metadata(geo_id: str) -> Optional[Dict]:
|
||||
"""
|
||||
Fetch GEO study metadata using NCBI Entrez E-utilities.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
Dict with study metadata or None if failed
|
||||
"""
|
||||
try:
|
||||
# Use esearch to get GEO UID
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term={geo_id}[Accession]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not id_list:
|
||||
logger.warning(f"No GEO entry found for {geo_id}")
|
||||
return None
|
||||
|
||||
# Use esummary to get metadata
|
||||
uid = id_list[0]
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gds&id={uid}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(uid, {})
|
||||
|
||||
return {
|
||||
'geo_id': geo_id,
|
||||
'title': result.get('title', 'N/A'),
|
||||
'summary': result.get('summary', 'N/A'),
|
||||
'organism': result.get('taxon', 'N/A'),
|
||||
'n_samples': result.get('n_samples', 0),
|
||||
'gpl': result.get('gpl', 'N/A'),
|
||||
'entrytype': result.get('entrytype', 'N/A'),
|
||||
'pubmed_ids': result.get('pubmedids', []),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching GEO metadata for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_study_accession(geo_id: str) -> Optional[str]:
|
||||
"""
|
||||
Get the SRA study accession (SRPxxxxxx) for a GEO accession.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
SRA study accession (e.g., 'SRP126328') or None
|
||||
"""
|
||||
try:
|
||||
# Search for SRA study linked to GEO
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not id_list:
|
||||
return None
|
||||
|
||||
# Get summary to extract SRP accession
|
||||
uid = id_list[0]
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=sra&id={uid}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(uid, {})
|
||||
exp_xml = result.get('expxml', '')
|
||||
|
||||
# Extract SRP from the XML
|
||||
srp_match = re.search(r'<Study acc="(SRP\d+)"', exp_xml)
|
||||
if srp_match:
|
||||
return srp_match.group(1)
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching SRA study for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_run_info(geo_id: str, bioproject: Optional[str] = None) -> List[Dict]:
|
||||
"""
|
||||
Fetch SRA run information for all samples in a GEO study.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
bioproject: Optional BioProject accession for fallback search
|
||||
|
||||
Returns:
|
||||
List of dicts with run info (srr, gsm, layout, library_strategy, etc.)
|
||||
"""
|
||||
runs = []
|
||||
|
||||
try:
|
||||
# First get the BioProject accession
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
# If no results, try BioProject fallback
|
||||
if not id_list:
|
||||
if not bioproject:
|
||||
bioproject = fetch_bioproject_from_geo(geo_id)
|
||||
|
||||
if bioproject:
|
||||
logger.info(f"Using BioProject {bioproject} for {geo_id}")
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={bioproject}&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
if not id_list:
|
||||
logger.warning(f"No SRA entries found for {geo_id}")
|
||||
return runs
|
||||
|
||||
# Batch fetch summaries
|
||||
ids_str = ','.join(id_list)
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=sra&id={ids_str}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=60)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=60) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {})
|
||||
|
||||
for uid in id_list:
|
||||
entry = result.get(uid, {})
|
||||
if not entry:
|
||||
continue
|
||||
|
||||
exp_xml = entry.get('expxml', '')
|
||||
runs_xml = entry.get('runs', '')
|
||||
|
||||
# Extract metadata from XML
|
||||
layout_match = re.search(r'<LIBRARY_LAYOUT>\s*<(\w+)', exp_xml)
|
||||
strategy_match = re.search(r'<LIBRARY_STRATEGY>(\w+)', exp_xml)
|
||||
source_match = re.search(r'<LIBRARY_SOURCE>(\w+)', exp_xml)
|
||||
gsm_match = re.search(r'<Sample acc="(GSM\d+)"', exp_xml)
|
||||
srx_match = re.search(r'<Experiment acc="(SRX\d+)"', exp_xml)
|
||||
|
||||
# Extract run accessions
|
||||
srr_matches = re.findall(r'<Run acc="(SRR\d+)"[^>]*total_spots="(\d+)"[^>]*total_bases="(\d+)"', runs_xml)
|
||||
|
||||
for srr, spots, bases in srr_matches:
|
||||
runs.append({
|
||||
'srr': srr,
|
||||
'srx': srx_match.group(1) if srx_match else '',
|
||||
'gsm': gsm_match.group(1) if gsm_match else '',
|
||||
'layout': layout_match.group(1).upper() if layout_match else 'UNKNOWN',
|
||||
'library_strategy': strategy_match.group(1) if strategy_match else 'UNKNOWN',
|
||||
'library_source': source_match.group(1) if source_match else 'UNKNOWN',
|
||||
'spots': int(spots),
|
||||
'bases': int(bases),
|
||||
})
|
||||
|
||||
return runs
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching SRA run info for {geo_id}: {e}")
|
||||
return runs
|
||||
|
||||
|
||||
def fetch_ena_fastq_urls(study_accession: str) -> Dict[str, List[str]]:
|
||||
"""
|
||||
Get FASTQ download URLs from ENA for an SRA study.
|
||||
|
||||
ENA provides faster downloads than SRA with pre-split paired files.
|
||||
|
||||
Args:
|
||||
study_accession: SRA study accession (e.g., 'SRP126328')
|
||||
|
||||
Returns:
|
||||
Dict mapping SRR accession to list of FASTQ URLs
|
||||
"""
|
||||
fastq_urls = {}
|
||||
|
||||
try:
|
||||
# Query ENA API
|
||||
ena_url = f"https://www.ebi.ac.uk/ena/portal/api/filereport?accession={study_accession}&result=read_run&fields=run_accession,sample_alias,fastq_ftp&format=tsv"
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(ena_url, timeout=60)
|
||||
content = response.text
|
||||
else:
|
||||
with urlopen(ena_url, timeout=60) as response:
|
||||
content = response.read().decode()
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
if len(lines) < 2:
|
||||
logger.warning(f"No FASTQ URLs found in ENA for {study_accession}")
|
||||
return fastq_urls
|
||||
|
||||
# Parse TSV
|
||||
header = lines[0].split('\t')
|
||||
run_idx = header.index('run_accession') if 'run_accession' in header else 0
|
||||
ftp_idx = header.index('fastq_ftp') if 'fastq_ftp' in header else 2
|
||||
|
||||
for line in lines[1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
fields = line.split('\t')
|
||||
if len(fields) > max(run_idx, ftp_idx):
|
||||
srr = fields[run_idx]
|
||||
ftp_urls = fields[ftp_idx]
|
||||
if ftp_urls:
|
||||
# URLs are semicolon-separated, convert to HTTP URLs
|
||||
# ENA supports both FTP and HTTP, HTTP is easier with requests
|
||||
urls = [f"http://{url}" for url in ftp_urls.split(';') if url]
|
||||
fastq_urls[srr] = urls
|
||||
|
||||
return fastq_urls
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching ENA URLs for {study_accession}: {e}")
|
||||
return fastq_urls
|
||||
|
||||
|
||||
def download_file(url: str, output_path: Path, timeout: int = 300, show_progress: bool = True) -> bool:
|
||||
"""
|
||||
Download a file with progress indication.
|
||||
|
||||
Args:
|
||||
url: URL to download
|
||||
output_path: Path to save file
|
||||
timeout: Download timeout in seconds
|
||||
show_progress: Show progress bar
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(url, stream=True, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
|
||||
with open(output_path, 'wb') as f:
|
||||
downloaded = 0
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
if show_progress and total_size > 0:
|
||||
pct = (downloaded / total_size) * 100
|
||||
print(f"\r Progress: {pct:.1f}%", end='', flush=True)
|
||||
if show_progress:
|
||||
print() # New line after progress
|
||||
return True
|
||||
else:
|
||||
# Fallback to urllib
|
||||
req = Request(url, headers={'User-Agent': 'geo-sra-skill/1.0'})
|
||||
with urlopen(req, timeout=timeout) as response:
|
||||
with open(output_path, 'wb') as f:
|
||||
shutil.copyfileobj(response, f)
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Download error for {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def fetch_pubmed_metadata(pmid: str, max_retries: int = 3) -> Optional[Dict]:
|
||||
"""
|
||||
Fetch paper metadata from PubMed.
|
||||
|
||||
Args:
|
||||
pmid: PubMed ID
|
||||
max_retries: Number of retries on failure
|
||||
|
||||
Returns:
|
||||
Dict with 'authors', 'year', 'journal', 'doi' or None
|
||||
"""
|
||||
url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id={pmid}&retmode=json"
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(pmid, {})
|
||||
|
||||
if not result or 'error' in result:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
return None
|
||||
|
||||
# Extract authors
|
||||
authors_list = result.get('authors', [])
|
||||
if not authors_list:
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
return None
|
||||
|
||||
author_names = [f"{a.get('name', '')}" for a in authors_list[:3]]
|
||||
authors = ', '.join(author_names)
|
||||
if len(authors_list) > 3:
|
||||
authors += ', et al.'
|
||||
|
||||
# Extract year
|
||||
pubdate = result.get('pubdate', '')
|
||||
year_match = re.search(r'\b(20\d{2})\b', pubdate)
|
||||
year = year_match.group(1) if year_match else "Unknown"
|
||||
|
||||
# Extract journal
|
||||
journal = result.get('source', 'Unknown')
|
||||
|
||||
# Extract DOI
|
||||
doi = ""
|
||||
for aid in result.get('articleids', []):
|
||||
if aid.get('idtype') == 'doi':
|
||||
doi = aid.get('value', '')
|
||||
break
|
||||
|
||||
return {
|
||||
'authors': authors,
|
||||
'year': year,
|
||||
'journal': journal,
|
||||
'doi': doi,
|
||||
'title': result.get('title', '')
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"PubMed fetch error for PMID {pmid} (attempt {attempt + 1}): {e}")
|
||||
if attempt < max_retries - 1:
|
||||
time.sleep(1 * (attempt + 1))
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""Format file size in human-readable format."""
|
||||
if size_bytes < 1024:
|
||||
return f"{size_bytes} B"
|
||||
elif size_bytes < 1024 * 1024:
|
||||
return f"{size_bytes / 1024:.1f} KB"
|
||||
elif size_bytes < 1024 * 1024 * 1024:
|
||||
return f"{size_bytes / (1024 * 1024):.1f} MB"
|
||||
else:
|
||||
return f"{size_bytes / (1024 * 1024 * 1024):.1f} GB"
|
||||
|
||||
|
||||
def estimate_download_size(runs: List[Dict]) -> int:
|
||||
"""
|
||||
Estimate total download size from SRA run info.
|
||||
|
||||
Args:
|
||||
runs: List of run info dicts with 'bases' field
|
||||
|
||||
Returns:
|
||||
Estimated size in bytes (rough estimate based on bases)
|
||||
"""
|
||||
total_bases = sum(r.get('bases', 0) for r in runs)
|
||||
# FASTQ is roughly 1 byte per base when compressed
|
||||
return total_bases // 4 # Rough compression ratio
|
||||
|
||||
|
||||
def fetch_bioproject_from_geo(geo_id: str) -> Optional[str]:
|
||||
"""
|
||||
Fetch BioProject accession linked to a GEO study.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
|
||||
Returns:
|
||||
BioProject accession (e.g., 'PRJNA432544') or None
|
||||
"""
|
||||
try:
|
||||
# First get GDS UID
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term={geo_id}[Accession]&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
gds_ids = data.get('esearchresult', {}).get('idlist', [])
|
||||
if not gds_ids:
|
||||
return None
|
||||
|
||||
# Get linked BioProject
|
||||
elink_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=gds&db=bioproject&id={gds_ids[0]}&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(elink_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(elink_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
linksets = data.get('linksets', [])
|
||||
if linksets and linksets[0].get('linksetdbs'):
|
||||
for linksetdb in linksets[0]['linksetdbs']:
|
||||
if linksetdb.get('dbto') == 'bioproject':
|
||||
bp_ids = linksetdb.get('links', [])
|
||||
if bp_ids:
|
||||
# Get BioProject accession
|
||||
summary_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=bioproject&id={bp_ids[0]}&retmode=json"
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(summary_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(summary_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
result = data.get('result', {}).get(str(bp_ids[0]), {})
|
||||
return result.get('project_acc')
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error fetching BioProject for {geo_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_sra_run_info_detailed(geo_id: str, bioproject: Optional[str] = None) -> List[Dict]:
|
||||
"""
|
||||
Fetch detailed SRA run information using efetch CSV format.
|
||||
|
||||
This provides richer metadata than esummary, including sample names.
|
||||
|
||||
Args:
|
||||
geo_id: GEO accession (e.g., 'GSE110004')
|
||||
bioproject: Optional BioProject accession for fallback search
|
||||
|
||||
Returns:
|
||||
List of dicts with detailed run info
|
||||
"""
|
||||
runs = []
|
||||
|
||||
try:
|
||||
# First get SRA UIDs using GEO search
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={geo_id}[GEO]&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
# If no results with GEO search, try BioProject
|
||||
if not id_list:
|
||||
# Try to find BioProject if not provided
|
||||
if not bioproject:
|
||||
logger.info(f"No direct SRA link for {geo_id}, searching for BioProject...")
|
||||
bioproject = fetch_bioproject_from_geo(geo_id)
|
||||
|
||||
if bioproject:
|
||||
logger.info(f"Found BioProject: {bioproject}")
|
||||
search_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=sra&term={bioproject}&retmax=1000&retmode=json"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(search_url, timeout=30)
|
||||
data = response.json()
|
||||
else:
|
||||
with urlopen(search_url, timeout=30) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
id_list = data.get('esearchresult', {}).get('idlist', [])
|
||||
|
||||
if not id_list:
|
||||
logger.warning(f"No SRA entries found for {geo_id}")
|
||||
return runs
|
||||
|
||||
# Fetch run info in CSV format using efetch
|
||||
ids_str = ','.join(id_list)
|
||||
efetch_url = f"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=sra&id={ids_str}&rettype=runinfo&retmode=csv"
|
||||
|
||||
_rate_limit_ncbi()
|
||||
if HAS_REQUESTS:
|
||||
response = requests.get(efetch_url, timeout=60)
|
||||
content = response.text
|
||||
else:
|
||||
with urlopen(efetch_url, timeout=60) as response:
|
||||
content = response.read().decode()
|
||||
|
||||
lines = content.strip().split('\n')
|
||||
if len(lines) < 1:
|
||||
return runs
|
||||
|
||||
# NCBI efetch runinfo CSV doesn't include headers
|
||||
# Define the fixed column order for SRA runinfo format
|
||||
header = [
|
||||
'Run', 'ReleaseDate', 'LoadDate', 'spots', 'bases', 'spots_with_mates',
|
||||
'avgLength', 'size_MB', 'AssemblyName', 'download_path', 'Experiment',
|
||||
'LibraryName', 'LibraryStrategy', 'LibrarySelection', 'LibrarySource',
|
||||
'LibraryLayout', 'InsertSize', 'InsertDev', 'Platform', 'Model',
|
||||
'SRAStudy', 'BioProject', 'Study_Pubmed_id', 'ProjectID', 'Sample',
|
||||
'BioSample', 'SampleType', 'TaxID', 'ScientificName', 'SampleName',
|
||||
'g1k_pop_code', 'source', 'g1k_analysis_group', 'Subject_ID', 'Sex',
|
||||
'Disease', 'Tumor', 'Affection_Status', 'Analyte_Type', 'Histological_Type',
|
||||
'Body_Site', 'CenterName', 'Submission', 'dbgap_study_accession', 'Consent',
|
||||
'RunHash', 'ReadHash'
|
||||
]
|
||||
|
||||
# Map column names to indices
|
||||
col_map = {col: idx for idx, col in enumerate(header)}
|
||||
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Handle CSV fields (some may contain commas in quotes)
|
||||
fields = _parse_csv_line(line)
|
||||
if len(fields) < len(header):
|
||||
continue
|
||||
|
||||
def get_field(name, default=''):
|
||||
idx = col_map.get(name, -1)
|
||||
return fields[idx] if idx >= 0 and idx < len(fields) else default
|
||||
|
||||
run = {
|
||||
'srr': get_field('Run'),
|
||||
'srx': get_field('Experiment'),
|
||||
'gsm': get_field('SampleName'), # Often GSM ID
|
||||
'sample_name': get_field('SampleName'),
|
||||
'library_name': get_field('LibraryName'),
|
||||
'layout': get_field('LibraryLayout', 'UNKNOWN').upper(),
|
||||
'library_strategy': get_field('LibraryStrategy', 'UNKNOWN'),
|
||||
'library_source': get_field('LibrarySource', 'UNKNOWN'),
|
||||
'library_selection': get_field('LibrarySelection', ''),
|
||||
'platform': get_field('Platform'),
|
||||
'model': get_field('Model'),
|
||||
'organism': get_field('ScientificName', ''),
|
||||
'spots': int(get_field('spots', 0) or 0),
|
||||
'bases': int(get_field('bases', 0) or 0),
|
||||
'size_mb': float(get_field('size_MB', 0) or 0),
|
||||
'bioproject': get_field('BioProject'),
|
||||
'biosample': get_field('BioSample'),
|
||||
'sra_study': get_field('SRAStudy'),
|
||||
}
|
||||
|
||||
# Only add if we have a valid SRR
|
||||
if run['srr'].startswith('SRR'):
|
||||
runs.append(run)
|
||||
|
||||
return runs
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching detailed SRA run info for {geo_id}: {e}")
|
||||
return runs
|
||||
|
||||
|
||||
def _parse_csv_line(line: str) -> List[str]:
|
||||
"""Parse a CSV line handling quoted fields."""
|
||||
import csv
|
||||
import io
|
||||
reader = csv.reader(io.StringIO(line))
|
||||
for row in reader:
|
||||
return row
|
||||
return []
|
||||
|
||||
|
||||
def group_samples_by_type(runs: List[Dict]) -> Dict[str, Dict]:
|
||||
"""
|
||||
Group SRA runs by library type and layout.
|
||||
|
||||
Returns dict with group names as keys and info dicts as values:
|
||||
{
|
||||
'RNA-Seq:PAIRED': {
|
||||
'runs': [...],
|
||||
'count': 18,
|
||||
'gsm_range': 'GSM2879618-GSM2879635',
|
||||
'size_estimate': 50000000000,
|
||||
'description': 'RNA-Seq paired-end'
|
||||
},
|
||||
...
|
||||
}
|
||||
"""
|
||||
groups = {}
|
||||
|
||||
for run in runs:
|
||||
strategy = run.get('library_strategy', 'UNKNOWN')
|
||||
layout = run.get('layout', 'UNKNOWN')
|
||||
key = f"{strategy}:{layout}"
|
||||
|
||||
if key not in groups:
|
||||
groups[key] = {
|
||||
'runs': [],
|
||||
'gsm_ids': set(),
|
||||
'total_bases': 0,
|
||||
'strategy': strategy,
|
||||
'layout': layout,
|
||||
}
|
||||
|
||||
groups[key]['runs'].append(run)
|
||||
gsm = run.get('gsm', '')
|
||||
if gsm.startswith('GSM'):
|
||||
groups[key]['gsm_ids'].add(gsm)
|
||||
groups[key]['total_bases'] += run.get('bases', 0)
|
||||
|
||||
# Post-process groups
|
||||
result = {}
|
||||
for key, info in groups.items():
|
||||
gsm_list = sorted(info['gsm_ids'])
|
||||
gsm_range = _format_gsm_range(gsm_list) if gsm_list else 'N/A'
|
||||
|
||||
result[key] = {
|
||||
'runs': info['runs'],
|
||||
'count': len(info['runs']),
|
||||
'gsm_range': gsm_range,
|
||||
'gsm_ids': gsm_list,
|
||||
'size_estimate': info['total_bases'] // 4, # Rough compressed size
|
||||
'strategy': info['strategy'],
|
||||
'layout': info['layout'],
|
||||
'description': f"{info['strategy']} {info['layout'].lower()}",
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def _format_gsm_range(gsm_list: List[str]) -> str:
|
||||
"""Format list of GSM IDs as a range if consecutive."""
|
||||
if not gsm_list:
|
||||
return 'N/A'
|
||||
|
||||
if len(gsm_list) == 1:
|
||||
return gsm_list[0]
|
||||
|
||||
# Extract numbers and check if consecutive
|
||||
try:
|
||||
numbers = [int(gsm.replace('GSM', '')) for gsm in gsm_list]
|
||||
numbers.sort()
|
||||
|
||||
if numbers[-1] - numbers[0] == len(numbers) - 1:
|
||||
# Consecutive
|
||||
return f"GSM{numbers[0]}-GSM{numbers[-1]}"
|
||||
else:
|
||||
# Not consecutive, show count
|
||||
return f"{gsm_list[0]}...({len(gsm_list)} samples)"
|
||||
except ValueError:
|
||||
return f"{len(gsm_list)} samples"
|
||||
|
||||
|
||||
def format_sample_groups_table(groups: Dict[str, Dict]) -> str:
|
||||
"""Format sample groups as a readable table."""
|
||||
lines = []
|
||||
lines.append("")
|
||||
lines.append(f"{'Sample Group':<20} {'Count':>6} {'Layout':<10} {'GSM Range':<25} {'Est. Size':>12}")
|
||||
lines.append("-" * 80)
|
||||
|
||||
for key, info in sorted(groups.items(), key=lambda x: -x[1]['count']):
|
||||
size_str = format_file_size(info['size_estimate'])
|
||||
lines.append(
|
||||
f"{info['strategy']:<20} {info['count']:>6} {info['layout']:<10} "
|
||||
f"{info['gsm_range']:<25} {size_str:>12}"
|
||||
)
|
||||
|
||||
lines.append("-" * 80)
|
||||
total_runs = sum(g['count'] for g in groups.values())
|
||||
total_size = sum(g['size_estimate'] for g in groups.values())
|
||||
lines.append(f"{'TOTAL':<20} {total_runs:>6} {'':<10} {'':<25} {format_file_size(total_size):>12}")
|
||||
|
||||
return '\n'.join(lines)
|
||||
@@ -0,0 +1,290 @@
|
||||
"""
|
||||
Sample name and metadata inference from filenames.
|
||||
|
||||
This module extracts sample information, detects tumor/normal status,
|
||||
and matches R1/R2 read pairs from sequencing file names.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
# R1/R2 patterns with priority scores (higher = more confident)
|
||||
R1_PATTERNS = [
|
||||
(r'_R1_\d{3}', 10), # _R1_001 (Illumina standard)
|
||||
(r'_R1[_.]', 8), # _R1. or _R1_
|
||||
(r'\.R1[_.]', 8), # .R1. or .R1_
|
||||
(r'_1[_.]', 5), # _1. or _1_
|
||||
(r'_R1\.f', 6), # _R1.fastq
|
||||
(r'_1\.f', 4), # _1.fastq
|
||||
]
|
||||
|
||||
R2_PATTERNS = [
|
||||
(r'_R2_\d{3}', 10), # _R2_001 (Illumina standard)
|
||||
(r'_R2[_.]', 8), # _R2. or _R2_
|
||||
(r'\.R2[_.]', 8), # .R2. or .R2_
|
||||
(r'_2[_.]', 5), # _2. or _2_
|
||||
(r'_R2\.f', 6), # _R2.fastq
|
||||
(r'_2\.f', 4), # _2.fastq
|
||||
]
|
||||
|
||||
# Tumor/normal keywords
|
||||
TUMOR_KEYWORDS = [
|
||||
r'\btumou?r\b',
|
||||
r'\bmetastasis\b',
|
||||
r'\bmet\b',
|
||||
r'\bprimary\b',
|
||||
r'\bcancer\b',
|
||||
r'\bmalignant\b',
|
||||
r'[-_]T[-_]',
|
||||
r'[-_]T\d*$',
|
||||
r'^T\d*[-_]',
|
||||
]
|
||||
|
||||
NORMAL_KEYWORDS = [
|
||||
r'\bnormal\b',
|
||||
r'\bgermline\b',
|
||||
r'\bblood\b',
|
||||
r'\bpbmc\b',
|
||||
r'\bcontrol\b',
|
||||
r'\bhealthy\b',
|
||||
r'\bmatched\b',
|
||||
r'[-_]N[-_]',
|
||||
r'[-_]N\d*$',
|
||||
r'^N\d*[-_]',
|
||||
]
|
||||
|
||||
# Lane pattern
|
||||
LANE_PATTERN = r'[_.]L(\d{3})[_.]'
|
||||
|
||||
# Patient/sample extraction patterns
|
||||
PATIENT_PATTERNS = [
|
||||
r'^(P\d+)[-_]', # P001_sample
|
||||
r'^(patient\d+)[-_]', # patient1_sample
|
||||
r'^(TCGA-\w+-\w+)', # TCGA format
|
||||
r'^([A-Z]{2,3}\d{3,})[-_]', # AB123_sample
|
||||
]
|
||||
|
||||
# Replicate patterns
|
||||
REPLICATE_PATTERNS = [
|
||||
r'[_.]rep(\d+)', # _rep1, .rep2
|
||||
r'[_.]replicate(\d+)', # _replicate1
|
||||
r'[_.]R(\d+)[_.]', # _R1_ (but not R1/R2 for reads!)
|
||||
r'[-_](\d+)$', # sample_1 (last resort)
|
||||
]
|
||||
|
||||
|
||||
def extract_sample_info(filepath: str) -> Dict[str, str]:
|
||||
"""
|
||||
Extract sample metadata from filepath.
|
||||
|
||||
Args:
|
||||
filepath: Path to sequencing file
|
||||
|
||||
Returns:
|
||||
Dict with: sample, patient, lane (if detectable)
|
||||
"""
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
# Remove extensions
|
||||
stem = filename
|
||||
for ext in ['.fastq.gz', '.fq.gz', '.fastq', '.fq', '.bam', '.cram', '.bai', '.crai']:
|
||||
if stem.lower().endswith(ext):
|
||||
stem = stem[:-len(ext)]
|
||||
break
|
||||
|
||||
info = {}
|
||||
|
||||
# Extract lane
|
||||
lane_match = re.search(LANE_PATTERN, stem)
|
||||
info['lane'] = f"L{lane_match.group(1)}" if lane_match else "L001"
|
||||
|
||||
# Remove lane from stem
|
||||
clean_stem = re.sub(LANE_PATTERN, '_', stem)
|
||||
|
||||
# Remove R1/R2 indicators and everything after
|
||||
for pattern, _ in R1_PATTERNS + R2_PATTERNS:
|
||||
clean_stem = re.sub(pattern + r'.*', '', clean_stem, flags=re.IGNORECASE)
|
||||
|
||||
# Clean up trailing/multiple underscores and dots
|
||||
clean_stem = re.sub(r'[_.-]+$', '', clean_stem)
|
||||
clean_stem = re.sub(r'[_.-]{2,}', '_', clean_stem)
|
||||
|
||||
# Try to extract patient ID
|
||||
for pattern in PATIENT_PATTERNS:
|
||||
match = re.match(pattern, clean_stem, re.IGNORECASE)
|
||||
if match:
|
||||
info['patient'] = match.group(1)
|
||||
break
|
||||
|
||||
# Sample is the cleaned stem
|
||||
info['sample'] = clean_stem if clean_stem else filename.split('.')[0]
|
||||
|
||||
# Default patient to sample if not extracted
|
||||
if 'patient' not in info:
|
||||
info['patient'] = info['sample']
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def infer_tumor_normal_status(sample_name: str) -> Optional[int]:
|
||||
"""
|
||||
Infer tumor (1) or normal (0) status from sample name.
|
||||
|
||||
Args:
|
||||
sample_name: Sample identifier
|
||||
|
||||
Returns:
|
||||
1 for tumor, 0 for normal, None if cannot determine
|
||||
"""
|
||||
name_lower = sample_name.lower()
|
||||
|
||||
# Check tumor indicators
|
||||
for pattern in TUMOR_KEYWORDS:
|
||||
if re.search(pattern, name_lower, re.IGNORECASE):
|
||||
return 1
|
||||
|
||||
# Check normal indicators
|
||||
for pattern in NORMAL_KEYWORDS:
|
||||
if re.search(pattern, name_lower, re.IGNORECASE):
|
||||
return 0
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def extract_replicate_number(sample_name: str) -> Optional[int]:
|
||||
"""
|
||||
Extract replicate number from sample name.
|
||||
|
||||
Args:
|
||||
sample_name: Sample identifier
|
||||
|
||||
Returns:
|
||||
Replicate number if found, None otherwise
|
||||
"""
|
||||
for pattern in REPLICATE_PATTERNS:
|
||||
match = re.search(pattern, sample_name, re.IGNORECASE)
|
||||
if match:
|
||||
try:
|
||||
return int(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _get_pattern_score(filename: str, patterns: List[Tuple[str, int]]) -> int:
|
||||
"""Get highest matching pattern score."""
|
||||
max_score = 0
|
||||
for pattern, score in patterns:
|
||||
if re.search(pattern, filename, re.IGNORECASE):
|
||||
max_score = max(max_score, score)
|
||||
return max_score
|
||||
|
||||
|
||||
def _get_sample_key(filepath: str) -> str:
|
||||
"""Generate a key for grouping related files."""
|
||||
info = extract_sample_info(filepath)
|
||||
sample = info['sample']
|
||||
lane = info.get('lane', 'L001')
|
||||
|
||||
# Include lane in key for multi-lane samples
|
||||
if lane != "L001":
|
||||
return f"{sample}_{lane}"
|
||||
return sample
|
||||
|
||||
|
||||
def match_read_pairs(files) -> Dict[str, Dict]:
|
||||
"""
|
||||
Match R1/R2 read pairs using scored pattern matching.
|
||||
|
||||
Args:
|
||||
files: List of FileInfo objects (from file_discovery)
|
||||
|
||||
Returns:
|
||||
Dict mapping sample_key to {'r1': path, 'r2': path, 'info': dict}
|
||||
"""
|
||||
# Classify files
|
||||
r1_files = []
|
||||
r2_files = []
|
||||
|
||||
for file in files:
|
||||
filename = file.name if hasattr(file, 'name') else os.path.basename(str(file))
|
||||
filepath = file.path if hasattr(file, 'path') else str(file)
|
||||
|
||||
r1_score = _get_pattern_score(filename, R1_PATTERNS)
|
||||
r2_score = _get_pattern_score(filename, R2_PATTERNS)
|
||||
|
||||
if r2_score > r1_score and r2_score > 0:
|
||||
r2_files.append((filepath, r2_score))
|
||||
elif r1_score > 0:
|
||||
r1_files.append((filepath, r1_score))
|
||||
else:
|
||||
# No clear indicator - assume R1 (single-end or non-standard naming)
|
||||
r1_files.append((filepath, 0))
|
||||
|
||||
# Build pairs by matching sample keys
|
||||
pairs = {}
|
||||
|
||||
# Process R1 files first
|
||||
for r1_path, score in r1_files:
|
||||
key = _get_sample_key(r1_path)
|
||||
info = extract_sample_info(r1_path)
|
||||
|
||||
if key not in pairs:
|
||||
pairs[key] = {
|
||||
'r1': r1_path,
|
||||
'r2': None,
|
||||
'info': info,
|
||||
'score': score
|
||||
}
|
||||
else:
|
||||
# Multiple R1 files for same sample (should not happen)
|
||||
pairs[key]['r1'] = r1_path
|
||||
|
||||
# Match R2 files
|
||||
for r2_path, score in r2_files:
|
||||
key = _get_sample_key(r2_path)
|
||||
info = extract_sample_info(r2_path)
|
||||
|
||||
if key in pairs:
|
||||
pairs[key]['r2'] = r2_path
|
||||
else:
|
||||
# R2 without matching R1
|
||||
pairs[key] = {
|
||||
'r1': None,
|
||||
'r2': r2_path,
|
||||
'info': info,
|
||||
'score': score
|
||||
}
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def infer_patient_groupings(sample_names: List[str]) -> Dict[str, str]:
|
||||
"""
|
||||
Infer patient groupings from sample names.
|
||||
|
||||
Groups samples that share a common prefix pattern.
|
||||
|
||||
Args:
|
||||
sample_names: List of sample identifiers
|
||||
|
||||
Returns:
|
||||
Dict mapping sample_name to patient_id
|
||||
"""
|
||||
patient_map = {}
|
||||
|
||||
for sample in sample_names:
|
||||
# Try to find a patient pattern
|
||||
for pattern in PATIENT_PATTERNS:
|
||||
match = re.match(pattern, sample, re.IGNORECASE)
|
||||
if match:
|
||||
patient_map[sample] = match.group(1)
|
||||
break
|
||||
|
||||
if sample not in patient_map:
|
||||
# Default: each sample is its own patient
|
||||
patient_map[sample] = sample
|
||||
|
||||
return patient_map
|
||||
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
Samplesheet validation utilities.
|
||||
|
||||
Validates samplesheet rows against pipeline configuration before writing,
|
||||
catching errors early with helpful messages.
|
||||
"""
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
import yaml
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
"""Result of samplesheet validation."""
|
||||
valid: bool
|
||||
errors: List[str] = field(default_factory=list)
|
||||
warnings: List[str] = field(default_factory=list)
|
||||
suggestions: List[str] = field(default_factory=list)
|
||||
|
||||
def __bool__(self):
|
||||
return self.valid
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Generate human-readable summary."""
|
||||
lines = []
|
||||
if self.errors:
|
||||
lines.append("Errors:")
|
||||
for e in self.errors:
|
||||
lines.append(f" - {e}")
|
||||
if self.warnings:
|
||||
lines.append("Warnings:")
|
||||
for w in self.warnings:
|
||||
lines.append(f" - {w}")
|
||||
if self.suggestions:
|
||||
lines.append("Suggestions:")
|
||||
for s in self.suggestions:
|
||||
lines.append(f" - {s}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def load_pipeline_config(pipeline: str) -> Optional[Dict]:
|
||||
"""Load pipeline configuration from YAML file."""
|
||||
# Find config directory relative to this file
|
||||
script_dir = Path(__file__).parent.parent.parent
|
||||
config_path = script_dir / "config" / "pipelines" / f"{pipeline}.yaml"
|
||||
|
||||
if not config_path.exists():
|
||||
return None
|
||||
|
||||
with open(config_path) as f:
|
||||
return yaml.safe_load(f)
|
||||
|
||||
|
||||
def validate_samplesheet(
|
||||
rows: List[Dict],
|
||||
pipeline: str,
|
||||
config: Optional[Dict] = None
|
||||
) -> ValidationResult:
|
||||
"""
|
||||
Validate samplesheet rows against pipeline requirements.
|
||||
|
||||
Args:
|
||||
rows: List of row dictionaries
|
||||
pipeline: Pipeline name (e.g., 'rnaseq', 'sarek')
|
||||
config: Optional pre-loaded config dict
|
||||
|
||||
Returns:
|
||||
ValidationResult with errors, warnings, and suggestions
|
||||
"""
|
||||
errors = []
|
||||
warnings = []
|
||||
suggestions = []
|
||||
|
||||
# Load config if not provided
|
||||
if config is None:
|
||||
config = load_pipeline_config(pipeline)
|
||||
|
||||
if config is None:
|
||||
errors.append(f"Unknown pipeline: {pipeline}")
|
||||
return ValidationResult(valid=False, errors=errors)
|
||||
|
||||
columns = config.get("samplesheet", {}).get("columns", [])
|
||||
required_cols = [c["name"] for c in columns if c.get("required", False)]
|
||||
|
||||
if not rows:
|
||||
errors.append("Samplesheet is empty - no samples found")
|
||||
return ValidationResult(valid=False, errors=errors)
|
||||
|
||||
# Validate each row
|
||||
for i, row in enumerate(rows):
|
||||
row_num = i + 2 # Account for header row
|
||||
|
||||
# Check required columns
|
||||
for col_name in required_cols:
|
||||
col_config = next((c for c in columns if c["name"] == col_name), None)
|
||||
|
||||
# Skip columns with conditions that don't apply
|
||||
if col_config and "condition" in col_config:
|
||||
# Simple condition check - skip for now
|
||||
# Full implementation would evaluate conditions
|
||||
pass
|
||||
|
||||
if col_name not in row or row[col_name] is None or row[col_name] == "":
|
||||
# Check if there's a default
|
||||
if col_config and "default" in col_config:
|
||||
continue
|
||||
errors.append(f"Row {row_num}: Missing required column '{col_name}'")
|
||||
|
||||
# Validate path columns exist
|
||||
for col_name in ["fastq_1", "fastq_2", "bam", "bai"]:
|
||||
if col_name in row and row[col_name]:
|
||||
path = row[col_name]
|
||||
if not os.path.exists(path):
|
||||
errors.append(f"Row {row_num}: File not found: {path}")
|
||||
elif not os.path.isfile(path):
|
||||
errors.append(f"Row {row_num}: Not a file: {path}")
|
||||
|
||||
# Validate enum values
|
||||
for col_config in columns:
|
||||
col_name = col_config["name"]
|
||||
if col_name in row and row[col_name] and "allowed" in col_config:
|
||||
value = row[col_name]
|
||||
allowed = col_config["allowed"]
|
||||
if value not in allowed:
|
||||
errors.append(
|
||||
f"Row {row_num}: Invalid value '{value}' for '{col_name}'. "
|
||||
f"Allowed: {allowed}"
|
||||
)
|
||||
|
||||
# Check R1/R2 pairing consistency
|
||||
r1 = row.get("fastq_1", "")
|
||||
r2 = row.get("fastq_2", "")
|
||||
if r1 and not r2:
|
||||
warnings.append(f"Row {row_num}: Single-end data (no R2 file)")
|
||||
elif r2 and not r1:
|
||||
errors.append(f"Row {row_num}: R2 present but R1 missing")
|
||||
|
||||
# Check for duplicate samples
|
||||
sample_col = "sample" if "sample" in rows[0] else "patient"
|
||||
if sample_col in rows[0]:
|
||||
samples = [r.get(sample_col, "") for r in rows]
|
||||
duplicates = [s for s in set(samples) if samples.count(s) > 1]
|
||||
if duplicates:
|
||||
warnings.append(f"Duplicate sample names: {duplicates}")
|
||||
suggestions.append(
|
||||
"Duplicates may be intentional (multi-lane sequencing). "
|
||||
"Verify sample grouping is correct."
|
||||
)
|
||||
|
||||
# Pipeline-specific validation
|
||||
if pipeline == "sarek":
|
||||
_validate_sarek_specific(rows, errors, warnings, suggestions)
|
||||
elif pipeline == "atacseq":
|
||||
_validate_atacseq_specific(rows, errors, warnings, suggestions)
|
||||
|
||||
return ValidationResult(
|
||||
valid=len(errors) == 0,
|
||||
errors=errors,
|
||||
warnings=warnings,
|
||||
suggestions=suggestions
|
||||
)
|
||||
|
||||
|
||||
def _validate_sarek_specific(
|
||||
rows: List[Dict],
|
||||
errors: List[str],
|
||||
warnings: List[str],
|
||||
suggestions: List[str]
|
||||
):
|
||||
"""Sarek-specific validation for tumor/normal pairing."""
|
||||
# Group by patient
|
||||
patients = {}
|
||||
for row in rows:
|
||||
patient = row.get("patient", "")
|
||||
status = row.get("status")
|
||||
|
||||
if patient not in patients:
|
||||
patients[patient] = {"tumor": 0, "normal": 0, "unknown": 0}
|
||||
|
||||
if status == 1:
|
||||
patients[patient]["tumor"] += 1
|
||||
elif status == 0:
|
||||
patients[patient]["normal"] += 1
|
||||
else:
|
||||
patients[patient]["unknown"] += 1
|
||||
|
||||
# Check pairing
|
||||
for patient, counts in patients.items():
|
||||
if counts["tumor"] > 0 and counts["normal"] == 0:
|
||||
warnings.append(
|
||||
f"Patient '{patient}': Tumor sample(s) without matched normal. "
|
||||
"Somatic calling works best with paired tumor-normal."
|
||||
)
|
||||
suggestions.append(
|
||||
f"For patient '{patient}': Add a normal sample or use tumor-only mode."
|
||||
)
|
||||
|
||||
if counts["unknown"] > 0:
|
||||
warnings.append(
|
||||
f"Patient '{patient}': {counts['unknown']} sample(s) with unknown status. "
|
||||
"Set status column to 0 (normal) or 1 (tumor)."
|
||||
)
|
||||
|
||||
|
||||
def _validate_atacseq_specific(
|
||||
rows: List[Dict],
|
||||
errors: List[str],
|
||||
warnings: List[str],
|
||||
suggestions: List[str]
|
||||
):
|
||||
"""ATAC-seq specific validation for replicates."""
|
||||
# Group by sample (condition)
|
||||
samples = {}
|
||||
for row in rows:
|
||||
sample = row.get("sample", "")
|
||||
replicate = row.get("replicate", 1)
|
||||
|
||||
if sample not in samples:
|
||||
samples[sample] = []
|
||||
|
||||
samples[sample].append(replicate)
|
||||
|
||||
# Check replicates
|
||||
for sample, reps in samples.items():
|
||||
if len(reps) < 2:
|
||||
warnings.append(
|
||||
f"Sample '{sample}': Only {len(reps)} replicate(s). "
|
||||
"Consensus peaks require 2+ replicates."
|
||||
)
|
||||
|
||||
# Check for duplicate replicate numbers
|
||||
if len(reps) != len(set(reps)):
|
||||
errors.append(
|
||||
f"Sample '{sample}': Duplicate replicate numbers detected. "
|
||||
"Each replicate must have a unique number."
|
||||
)
|
||||
|
||||
# Check all samples have R2 (ATAC-seq requires paired-end)
|
||||
for i, row in enumerate(rows):
|
||||
if not row.get("fastq_2"):
|
||||
errors.append(
|
||||
f"Row {i+2}: ATAC-seq requires paired-end data. R2 file missing."
|
||||
)
|
||||
|
||||
|
||||
def validate_file_exists(path: str) -> bool:
|
||||
"""Check if file exists and is accessible."""
|
||||
return os.path.isfile(path) and os.access(path, os.R_OK)
|
||||
|
||||
|
||||
def validate_absolute_path(path: str) -> bool:
|
||||
"""Check if path is absolute."""
|
||||
return os.path.isabs(path)
|
||||
Reference in New Issue
Block a user