Setting the file. One moment. Manage Genomes · Nextflow Development · anthropics/knowledge-work-plugins · Skills Docs22
Validate Data
Tech Debt
62
Recruiting Pipeline
71
Vendor Check
125
Zoom Meeting SDK Web
88
Vendor Review
181
Create An Asset
Video Sdk/web
scripts/manage_genomes.py
Python·521 lines·16 KB
json
17import os
18import subprocess
19import sys
20from pathlib import Path
21from typing import Dict, List, Optional
22
23
24# iGenomes reference configuration
25IGENOMES = {
26 # Human
27 'GRCh38': {
28 'display_name': 'Human GRCh38/hg38',
29 'species': 'Homo sapiens',
30 'aliases': ['hg38', 'GRCh38.p14'],
31 's3_base': 's3://ngi-igenomes/igenomes/Homo_sapiens/NCBI/GRCh38',
32 'files': {
33 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
34 'gtf': 'Annotation/Genes/genes.gtf',
35 'bwa_index': 'Sequence/BWAIndex/',
36 'star_index': 'Sequence/STARIndex/',
37 }
38 },
39 'GRCh37': {
40 'display_name': 'Human GRCh37/hg19',
41 'species': 'Homo sapiens',
42 'aliases': ['hg19', 'GRCh37.p13'],
43 's3_base': 's3://ngi-igenomes/igenomes/Homo_sapiens/NCBI/GRCh37',
44 'files': {
45 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
46 'gtf': 'Annotation/Genes/genes.gtf',
47 'bwa_index': 'Sequence/BWAIndex/',
48 'star_index': 'Sequence/STARIndex/',
49 }
50 },
51 # Mouse
52 'GRCm39': {
53 'display_name': 'Mouse GRCm39/mm39',
54 'species': 'Mus musculus',
55 'aliases': ['mm39', 'GRCm39'],
56 's3_base': 's3://ngi-igenomes/igenomes/Mus_musculus/Ensembl/GRCm39',
57 'files': {
58 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
59 'gtf': 'Annotation/Genes/genes.gtf',
60 'bwa_index': 'Sequence/BWAIndex/',
61 'star_index': 'Sequence/STARIndex/',
62 }
63 },
64 'GRCm38': {
65 'display_name': 'Mouse GRCm38/mm10',
66 'species': 'Mus musculus',
67 'aliases': ['mm10', 'GRCm38'],
68 's3_base': 's3://ngi-igenomes/igenomes/Mus_musculus/NCBI/GRCm38',
69 'files': {
70 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
71 'gtf': 'Annotation/Genes/genes.gtf',
72 'bwa_index': 'Sequence/BWAIndex/',
73 'star_index': 'Sequence/STARIndex/',
74 }
75 },
76 # Yeast
77 'R64-1-1': {
78 'display_name': 'Yeast R64-1-1/sacCer3',
79 'species': 'Saccharomyces cerevisiae',
80 'aliases': ['sacCer3', 'S288C', 'yeast'],
81 's3_base': 's3://ngi-igenomes/igenomes/Saccharomyces_cerevisiae/Ensembl/R64-1-1',
82 'files': {
83 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
84 'gtf': 'Annotation/Genes/genes.gtf',
85 'bwa_index': 'Sequence/BWAIndex/',
86 'star_index': 'Sequence/STARIndex/',
87 }
88 },
89 # Fruit fly
90 'BDGP6': {
91 'display_name': 'Drosophila BDGP6/dm6',
92 'species': 'Drosophila melanogaster',
93 'aliases': ['dm6', 'BDGP6', 'fly'],
94 's3_base': 's3://ngi-igenomes/igenomes/Drosophila_melanogaster/Ensembl/BDGP6',
95 'files': {
96 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
97 'gtf': 'Annotation/Genes/genes.gtf',
98 }
99 },
100 # C. elegans
101 'WBcel235': {
102 'display_name': 'C. elegans WBcel235/ce11',
103 'species': 'Caenorhabditis elegans',
104 'aliases': ['ce11', 'worm'],
105 's3_base': 's3://ngi-igenomes/igenomes/Caenorhabditis_elegans/Ensembl/WBcel235',
106 'files': {
107 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
108 'gtf': 'Annotation/Genes/genes.gtf',
109 'bwa_index': 'Sequence/BWAIndex/',
110 'star_index': 'Sequence/STARIndex/',
111 }
112 },
113 # Zebrafish
114 'GRCz11': {
115 'display_name': 'Zebrafish GRCz11/danRer11',
116 'species': 'Danio rerio',
117 'aliases': ['danRer11', 'zebrafish'],
118 's3_base': 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz11',
119 'files': {
120 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
121 'gtf': 'Annotation/Genes/genes.gtf',
122 'bwa_index': 'Sequence/BWAIndex/',
123 'star_index': 'Sequence/STARIndex/',
124 }
125 },
126 'GRCz10': {
127 'display_name': 'Zebrafish GRCz10/danRer10',
128 'species': 'Danio rerio',
129 'aliases': ['danRer10'],
130 's3_base': 's3://ngi-igenomes/igenomes/Danio_rerio/Ensembl/GRCz10',
131 'files': {
132 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
133 'gtf': 'Annotation/Genes/genes.gtf',
134 }
135 },
136 # Rat
137 'Rnor_6.0': {
138 'display_name': 'Rat Rnor_6.0/rn6',
139 'species': 'Rattus norvegicus',
140 'aliases': ['rn6', 'Rnor6', 'rat'],
141 's3_base': 's3://ngi-igenomes/igenomes/Rattus_norvegicus/Ensembl/Rnor_6.0',
142 'files': {
143 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
144 'gtf': 'Annotation/Genes/genes.gtf',
145 'bwa_index': 'Sequence/BWAIndex/',
146 'star_index': 'Sequence/STARIndex/',
147 }
148 },
149 # Arabidopsis
150 'TAIR10': {
151 'display_name': 'Arabidopsis TAIR10',
152 'species': 'Arabidopsis thaliana',
153 'aliases': ['arabidopsis'],
154 's3_base': 's3://ngi-igenomes/igenomes/Arabidopsis_thaliana/Ensembl/TAIR10',
155 'files': {
156 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
157 'gtf': 'Annotation/Genes/genes.gtf',
158 'bwa_index': 'Sequence/BWAIndex/',
159 'star_index': 'Sequence/STARIndex/',
160 }
161 },
162 # Chicken
163 'GRCg6a': {
164 'display_name': 'Chicken GRCg6a/galGal6',
165 'species': 'Gallus gallus',
166 'aliases': ['galGal6', 'chicken'],
167 's3_base': 's3://ngi-igenomes/igenomes/Gallus_gallus/Ensembl/GRCg6a',
168 'files': {
169 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
170 'gtf': 'Annotation/Genes/genes.gtf',
171 }
172 },
173 # Dog
174 'CanFam3.1': {
175 'display_name': 'Dog CanFam3.1/canFam3',
176 'species': 'Canis lupus familiaris',
177 'aliases': ['canFam3', 'dog'],
178 's3_base': 's3://ngi-igenomes/igenomes/Canis_familiaris/Ensembl/CanFam3.1',
179 'files': {
180 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
181 'gtf': 'Annotation/Genes/genes.gtf',
182 }
183 },
184 # Pig
185 'Sscrofa11.1': {
186 'display_name': 'Pig Sscrofa11.1/susScr11',
187 'species': 'Sus scrofa',
188 'aliases': ['susScr11', 'pig'],
189 's3_base': 's3://ngi-igenomes/igenomes/Sus_scrofa/Ensembl/Sscrofa11.1',
190 'files': {
191 'fasta': 'Sequence/WholeGenomeFasta/genome.fa',
192 'gtf': 'Annotation/Genes/genes.gtf',
193 }
194 },
195}
196
197
198def get_cache_dir() -> Path:
199 """Get genome cache directory."""
200 cache_dir = os.environ.get(
201 'NF_CORE_GENOME_CACHE',
202 os.path.expanduser('~/.nf-core/genomes')
203 )
204 return Path(cache_dir)
205
206
207def resolve_genome_id(genome: str) -> Optional[str]:
208 """Resolve genome ID from name or alias."""
209 # Direct match
210 if genome in IGENOMES:
211 return genome
212
213 # Check aliases
214 genome_lower = genome.lower()
215 for gid, info in IGENOMES.items():
216 if genome_lower in [a.lower() for a in info.get('aliases', [])]:
217 return gid
218
219 return None
220
221
222def is_genome_installed(genome_id: str) -> bool:
223 """Check if genome is installed locally."""
224 cache_dir = get_cache_dir()
225 genome_dir = cache_dir / genome_id
226
227 # Check for fasta as minimum requirement
228 fasta_path = genome_dir / 'genome.fa'
229 return fasta_path.exists()
230
231
232def get_genome_path(genome_id: str) -> Optional[Path]:
233 """Get local path to genome if installed."""
234 if not is_genome_installed(genome_id):
235 return None
236 return get_cache_dir() / genome_id
237
238
239def list_genomes(installed_only: bool = False) -> List[Dict]:
240 """List available genomes."""
241 result = []
242
243 for genome_id, info in IGENOMES.items():
244 installed = is_genome_installed(genome_id)
245
246 if installed_only and not installed:
247 continue
248
249 genome_path = get_genome_path(genome_id) if installed else None
250
251 result.append({
252 'id': genome_id,
253 'display_name': info['display_name'],
254 'species': info['species'],
255 'aliases': info.get('aliases', []),
256 'installed': installed,
257 'path': str(genome_path) if genome_path else None,
258 })
259
260 return result
261
262
263def download_genome(
264 genome_id: str,
265 components: Optional[List[str]] = None,
266 force: bool = False
267) -> bool:
268 """
269 Download genome reference files from iGenomes.
270
271 Args:
272 genome_id: Genome identifier (e.g., GRCh38)
273 components: Specific components to download (fasta, gtf, etc.)
274 force: Overwrite existing files
275
276 Returns:
277 True if successful
278 """
279 # Resolve genome ID
280 resolved = resolve_genome_id(genome_id)
281 if not resolved:
282 print(f"Unknown genome: {genome_id}")
283 print(f"Available: {', '.join(IGENOMES.keys())}")
284 return False
285
286 genome_id = resolved
287 info = IGENOMES[genome_id]
288
289 # Check for AWS CLI
290 aws_available = subprocess.run(
291 ['which', 'aws'],
292 capture_output=True
293 ).returncode == 0
294
295 if not aws_available:
296 print("AWS CLI not found. Required for iGenomes download.")
297 print("Install with: pip install awscli")
298 print("\nAlternative: Use --genome flag with nf-core pipelines")
299 print("which will auto-download references (slower, per-run).")
300 return False
301
302 # Create cache directory
303 cache_dir = get_cache_dir()
304 genome_dir = cache_dir / genome_id
305 genome_dir.mkdir(parents=True, exist_ok=True)
306
307 # Determine components to download
308 if components is None:
309 components = ['fasta', 'gtf'] # Minimum required
310
311 print(f"Downloading {info['display_name']} to {genome_dir}")
312 print(f"Components: {', '.join(components)}")
313
314 success = True
315 for component in components:
316 if component not in info.get('files', {}):
317 print(f" Skipping {component}: not available for {genome_id}")
318 continue
319
320 remote_path = info['files'][component]
321 s3_path = f"{info['s3_base']}/{remote_path}"
322
323 # Determine local path
324 if remote_path.endswith('/'):
325 # Directory (e.g., index)
326 local_path = genome_dir / component
327 else:
328 # File
329 filename = Path(remote_path).name
330 local_path = genome_dir / filename
331
332 if local_path.exists() and not force:
333 print(f" {component}: Already exists (use --force to overwrite)")
334 continue
335
336 print(f" Downloading {component}...")
337
338 # Build AWS command
339 cmd = ['aws', 's3', 'cp', '--no-sign-request']
340
341 if remote_path.endswith('/'):
342 cmd.extend(['--recursive', s3_path, str(local_path)])
343 else:
344 cmd.extend([s3_path, str(local_path)])
345
346 result = subprocess.run(cmd, capture_output=True, text=True)
347
348 if result.returncode != 0:
349 print(f" ERROR downloading {component}:")
350 print(f" {result.stderr[:200]}")
351 success = False
352 else:
353 print(f" {component}: Downloaded successfully")
354
355 if success:
356 print(f"\nGenome {genome_id} ready at: {genome_dir}")
357 else:
358 print(f"\nSome components failed to download.")
359
360 return success
361
362
363def get_nextflow_params(genome_id: str) -> Dict[str, str]:
364 """
365 Get Nextflow parameters for a genome.
366
367 Returns dict with --fasta, --gtf if local,
368 or just --genome if using iGenomes key.
369 """
370 resolved = resolve_genome_id(genome_id)
371 if not resolved:
372 return {'error': f'Unknown genome: {genome_id}'}
373
374 genome_id = resolved
375
376 # Check if installed locally
377 genome_path = get_genome_path(genome_id)
378
379 if genome_path:
380 params = {}
381
382 # Check for local files
383 fasta = genome_path / 'genome.fa'
384 if fasta.exists():
385 params['fasta'] = str(fasta)
386
387 gtf = genome_path / 'genes.gtf'
388 if gtf.exists():
389 params['gtf'] = str(gtf)
390
391 if params:
392 return params
393
394 # Fall back to iGenomes key
395 return {'genome': genome_id}
396
397
398def print_genome_list(genomes: List[Dict], output_json: bool = False):
399 """Print genome list."""
400 if output_json:
401 print(json.dumps(genomes, indent=2))
402 return
403
404 print("\n" + "=" * 50)
405 print(" Available Genomes")
406 print("=" * 50 + "\n")
407
408 for g in genomes:
409 status = "\033[92m[installed]\033[0m" if g['installed'] else ""
410 print(f" {g['id']}: {g['display_name']} {status}")
411 print(f" Species: {g['species']}")
412 print(f" Aliases: {', '.join(g['aliases'])}")
413 if g['path']:
414 print(f" Path: {g['path']}")
415 print()
416
417
418def main():
419 parser = argparse.ArgumentParser(
420 description='Manage genome references for nf-core pipelines',
421 formatter_class=argparse.RawDescriptionHelpFormatter,
422 epilog="""
423Commands:
424 list List available genomes
425 check <genome> Check if genome is installed
426 download <genome> Download genome from iGenomes
427 params <genome> Get Nextflow parameters for genome
428
429Examples:
430 %(prog)s list
431 %(prog)s list --installed
432 %(prog)s check GRCh38
433 %(prog)s download GRCh38
434 %(prog)s download GRCh38 --components fasta gtf star_index
435 %(prog)s params GRCh38
436 """
437 )
438
439 subparsers = parser.add_subparsers(dest='command', help='Commands')
440
441 # List command
442 list_parser = subparsers.add_parser('list', help='List available genomes')
443 list_parser.add_argument('--installed', action='store_true',
444 help='Show only installed genomes')
445 list_parser.add_argument('--json', action='store_true',
446 help='Output as JSON')
447
448 # Check command
449 check_parser = subparsers.add_parser('check', help='Check if genome is installed')
450 check_parser.add_argument('genome', help='Genome ID (e.g., GRCh38)')
451 check_parser.add_argument('--json', action='store_true',
452 help='Output as JSON')
453
454 # Download command
455 dl_parser = subparsers.add_parser('download', help='Download genome from iGenomes')
456 dl_parser.add_argument('genome', help='Genome ID (e.g., GRCh38)')
457 dl_parser.add_argument('--components', nargs='+',
458 help='Specific components (fasta, gtf, bwa_index, star_index)')
459 dl_parser.add_argument('--force', action='store_true',
460 help='Overwrite existing files')
461
462 # Params command
463 params_parser = subparsers.add_parser('params', help='Get Nextflow params for genome')
464 params_parser.add_argument('genome', help='Genome ID')
465 params_parser.add_argument('--json', action='store_true',
466 help='Output as JSON')
467
468 args = parser.parse_args()
469
470 if args.command == 'list':
471 genomes = list_genomes(installed_only=args.installed)
472 print_genome_list(genomes, args.json)
473
474 elif args.command == 'check':
475 resolved = resolve_genome_id(args.genome)
476 if not resolved:
477 print(f"Unknown genome: {args.genome}")
478 sys.exit(1)
479
480 installed = is_genome_installed(resolved)
481 path = get_genome_path(resolved) if installed else None
482
483 if args.json:
484 print(json.dumps({
485 'genome': resolved,
486 'installed': installed,
487 'path': str(path) if path else None
488 }))
489 else:
490 if installed:
491 print(f"✓ Genome {resolved} is installed at: {path}")
492 else:
493 print(f"✗ Genome {resolved} is not installed locally")
494 print(f" Download with: python {sys.argv[0]} download {resolved}")
495
496 sys.exit(0 if installed else 1)
497
498 elif args.command == 'download':
499 success = download_genome(args.genome, args.components, args.force)
500 sys.exit(0 if success else 1)
501
502 elif args.command == 'params':
503 params = get_nextflow_params(args.genome)
504
505 if args.json:
506 print(json.dumps(params))
507 else:
508 if 'error' in params:
509 print(f"Error: {params['error']}")
510 sys.exit(1)
511
512 for key, value in params.items():
513 print(f"--{key} {value}")
514
515 else:
516 parser.print_help()
517 sys.exit(1)
518
519
520if __name__ == '__main__':
521 main()