Setting the file. One moment.
Sync · Pinecone Assistant · pinecone-io/skills · Skills Docs
Back to the top of the page scripts/ sync.py
Python · 359 lines · 13 KB
17
PINECONE_API_KEY: Required Pinecone API key
18
19 Output:
20 Shows files to add, update, and optionally delete, with confirmation prompt
21 """
22
23 import os
24 import hashlib
25 from pathlib import Path
26 from datetime import datetime, timezone
27 import typer
28 from rich.console import Console
29 from rich.panel import Panel
30 from rich.table import Table
31 from rich.progress import Progress, SpinnerColumn, TextColumn
32 from pinecone import Pinecone
33
34 app = typer.Typer()
35 console = Console()
36
37 # Supported file extensions
38 SUPPORTED_EXTENSIONS = { '.md' , '.txt' , '.pdf' , '.docx' , '.json' }
39
40 # Directories to exclude
41 EXCLUDE_DIRS = { 'node_modules' , '.venv' , '.git' , 'build' , 'dist' , '__pycache__' , '.pytest_cache' }
42
43
44 def should_exclude_path (path: Path, source_root: Path) -> bool :
45 """Check if path should be excluded based on directory patterns."""
46 try :
47 rel_path = path.relative_to(source_root)
48 for part in rel_path.parts:
49 if part in EXCLUDE_DIRS or part.startswith( '.' ):
50 return True
51 except ValueError :
52 return True
53 return False
54
55
56 def find_files (source_path: Path) -> list[Path]:
57 """Find all supported files in source directory, excluding common build/dependency dirs."""
58 files = []
59
60 if source_path.is_file():
61 if source_path.suffix.lower() in SUPPORTED_EXTENSIONS :
62 return [source_path]
63 else :
64 return []
65
66 for file_path in source_path.rglob( '*' ):
67 if file_path.is_file():
68 if file_path.suffix.lower() in SUPPORTED_EXTENSIONS :
69 if not should_exclude_path(file_path, source_path):
70 files.append(file_path)
71
72 return sorted (files)
73
74
75 def get_file_info (file_path: Path):
76 """Get file modification time and size."""
77 stat = file_path.stat()
78 return {
79 'mtime' : stat.st_mtime,
80 'size' : stat.st_size,
81 }
82
83
84 def file_changed (local_info: dict , remote_metadata: dict ) -> bool :
85 """Check if local file differs from remote using mtime and size."""
86 remote_mtime = remote_metadata.get( 'mtime' )
87 remote_size = remote_metadata.get( 'size' )
88
89 if remote_mtime is None or remote_size is None :
90 # No stored metadata, assume changed
91 return True
92
93 return (local_info[ 'mtime' ] != float (remote_mtime) or
94 local_info[ 'size' ] != int (remote_size))
95
96
97 @app.command ()
98 def main (
99 assistant: str = typer.Option( ... , "--assistant" , "-a" , help = "Name of the assistant" ),
100 source: str = typer.Option( ... , "--source" , "-s" , help = "Local file or directory path" ),
101 delete_missing: bool = typer.Option( False , "--delete-missing" , help = "Delete files from assistant that don't exist locally" ),
102 dry_run: bool = typer.Option( False , "--dry-run" , help = "Show what would change without making changes" ),
103 yes: bool = typer.Option( False , "--yes" , "-y" , help = "Skip confirmation prompt" ),
104 ):
105 """Sync local files to Pinecone Assistant, only uploading new or changed files."""
106
107 # Check for API key
108 api_key = os.environ.get( "PINECONE_API_KEY" )
109 if not api_key:
110 console.print( "[red]Error: PINECONE_API_KEY environment variable not set[/red]" )
111 console.print( " \n Get your API key from: https://app.pinecone.io/?sessionType=signup" )
112 raise typer.Exit( 1 )
113
114 # Validate source path
115 source_path = Path(source).resolve()
116 if not source_path.exists():
117 console.print( f "[red]Error: Source path does not exist: { source } [/red]" )
118 raise typer.Exit( 1 )
119
120 try :
121 # Initialize Pinecone client
122 pc = Pinecone( api_key = api_key, source_tag = "pinecone_skills:assistant" )
123 # describe() returns a model carrying a client back-reference, so the
124 # asst.list_files()/upload_file()/delete_file() calls below keep working.
125 # Its list_files() also materializes, unlike pc.assistants.list_files(),
126 # which returns a Paginator with no __len__.
127 asst = pc.assistants.describe( name = assistant)
128
129 console.print(Panel(
130 f "[bold cyan]Assistant:[/bold cyan] { assistant }\n "
131 f "[bold cyan]Source:[/bold cyan] { source_path } " ,
132 title = "Sync Configuration" ,
133 border_style = "cyan"
134 ))
135
136 # Step 1: Get current files in assistant
137 with console.status( "[bold blue]Fetching assistant files...[/bold blue]" , spinner = "dots" ):
138 remote_files = asst.list_files()
139
140 # Build map of file_path -> file object
141 remote_file_map = {}
142 for f in remote_files:
143 metadata = getattr (f, 'metadata' , {}) or {}
144 file_path = metadata.get( 'file_path' , f.name)
145 remote_file_map[file_path] = {
146 'file_obj' : f,
147 'metadata' : metadata
148 }
149
150 console.print( f "[dim]Found { len (remote_files) } file(s) in assistant[/dim] \n " )
151
152 # Step 2: Find local files
153 with console.status( "[bold blue]Scanning local files...[/bold blue]" , spinner = "dots" ):
154 local_files = find_files(source_path)
155
156 if not local_files:
157 console.print( "[yellow]No supported files found in source path[/yellow]" )
158 console.print( f "Supported extensions: { ', ' .join( sorted ( SUPPORTED_EXTENSIONS )) } " )
159 raise typer.Exit( 0 )
160
161 console.print( f "[dim]Found { len (local_files) } local file(s)[/dim] \n " )
162
163 # Step 3: Determine what needs syncing
164 to_upload = [] # New files
165 to_update = [] # Changed files (delete + re-upload)
166 to_delete = [] # Files in assistant but not local
167 unchanged = [] # Files that match
168
169 # Track which remote files we've seen
170 seen_remote_paths = set ()
171
172 for local_file in local_files:
173 # Get relative path from source root
174 if source_path.is_file():
175 rel_path = local_file.name
176 else :
177 rel_path = str (local_file.relative_to(source_path))
178
179 local_info = get_file_info(local_file)
180
181 if rel_path in remote_file_map:
182 # File exists remotely, check if changed
183 seen_remote_paths.add(rel_path)
184 remote_info = remote_file_map[rel_path]
185
186 if file_changed(local_info, remote_info[ 'metadata' ]):
187 to_update.append({
188 'local_path' : local_file,
189 'rel_path' : rel_path,
190 'remote_file_id' : remote_info[ 'file_obj' ].id,
191 'local_info' : local_info
192 })
193 else :
194 unchanged.append(rel_path)
195 else :
196 # New file
197 to_upload.append({
198 'local_path' : local_file,
199 'rel_path' : rel_path,
200 'local_info' : local_info
201 })
202
203 # Find files to delete (in remote but not local)
204 if delete_missing:
205 for rel_path, remote_info in remote_file_map.items():
206 if rel_path not in seen_remote_paths:
207 to_delete.append({
208 'rel_path' : rel_path,
209 'remote_file_id' : remote_info[ 'file_obj' ].id
210 })
211
212 # Step 4: Show summary
213 console.print( "[bold]Sync Summary:[/bold] \n " )
214
215 summary_table = Table( show_header = True , header_style = "bold cyan" )
216 summary_table.add_column( "Action" , style = "yellow" , width = 15 )
217 summary_table.add_column( "Count" , style = "green" , width = 10 )
218
219 summary_table.add_row( "New files" , str ( len (to_upload)))
220 summary_table.add_row( "Updated files" , str ( len (to_update)))
221 if delete_missing:
222 summary_table.add_row( "Deleted files" , str ( len (to_delete)))
223 summary_table.add_row( "Unchanged" , str ( len (unchanged)))
224
225 console.print(summary_table)
226 console.print()
227
228 # Show details if there are changes
229 if to_upload:
230 console.print( "[bold green]Files to upload:[/bold green]" )
231 for item in to_upload[: 10 ]: # Show first 10
232 console.print( f " + { item[ 'rel_path' ] } " )
233 if len (to_upload) > 10 :
234 console.print( f " ... and { len (to_upload) - 10 } more" )
235 console.print()
236
237 if to_update:
238 console.print( "[bold yellow]Files to update:[/bold yellow]" )
239 for item in to_update[: 10 ]:
240 console.print( f " ~ { item[ 'rel_path' ] } " )
241 if len (to_update) > 10 :
242 console.print( f " ... and { len (to_update) - 10 } more" )
243 console.print()
244
245 if to_delete:
246 console.print( "[bold red]Files to delete:[/bold red]" )
247 for item in to_delete[: 10 ]:
248 console.print( f " - { item[ 'rel_path' ] } " )
249 if len (to_delete) > 10 :
250 console.print( f " ... and { len (to_delete) - 10 } more" )
251 console.print()
252
253 # If no changes, exit early
254 if not (to_upload or to_update or to_delete):
255 console.print( "[green]✓ All files are up to date![/green]" )
256 return
257
258 # Dry run mode
259 if dry_run:
260 console.print( "[yellow]Dry run mode: No changes made[/yellow]" )
261 return
262
263 # Confirmation prompt
264 if not yes:
265 proceed = typer.confirm( " \n Proceed with sync?" )
266 if not proceed:
267 console.print( "[yellow]Sync cancelled[/yellow]" )
268 return
269
270 console.print()
271
272 # Step 5: Execute sync
273 uploaded_count = 0
274 updated_count = 0
275 deleted_count = 0
276
277 with Progress(
278 SpinnerColumn(),
279 TextColumn( "[progress.description] {task.description} " ),
280 console = console
281 ) as progress:
282
283 # Upload new files
284 if to_upload:
285 task = progress.add_task( f "Uploading { len (to_upload) } new file(s)..." , total = len (to_upload))
286 for item in to_upload:
287 try :
288 asst.upload_file(
289 file_path = str (item[ 'local_path' ]),
290 metadata = {
291 'file_path' : item[ 'rel_path' ],
292 'mtime' : item[ 'local_info' ][ 'mtime' ],
293 'size' : item[ 'local_info' ][ 'size' ],
294 'uploaded_at' : datetime.now(timezone.utc).isoformat(),
295 'source' : 'sync_script' ,
296 },
297 timeout = None
298 )
299 uploaded_count += 1
300 progress.advance(task)
301 except Exception as e:
302 console.print( f "[red]Failed to upload { item[ 'rel_path' ] } : { e } [/red]" )
303
304 # Update changed files (delete old + upload new)
305 if to_update:
306 task = progress.add_task( f "Updating { len (to_update) } file(s)..." , total = len (to_update) * 2 )
307 for item in to_update:
308 try :
309 # Delete old version
310 asst.delete_file( file_id = item[ 'remote_file_id' ])
311 progress.advance(task)
312
313 # Upload new version
314 asst.upload_file(
315 file_path = str (item[ 'local_path' ]),
316 metadata = {
317 'file_path' : item[ 'rel_path' ],
318 'mtime' : item[ 'local_info' ][ 'mtime' ],
319 'size' : item[ 'local_info' ][ 'size' ],
320 'uploaded_at' : datetime.now(timezone.utc).isoformat(),
321 'source' : 'sync_script' ,
322 },
323 timeout = None
324 )
325 updated_count += 1
326 progress.advance(task)
327 except Exception as e:
328 console.print( f "[red]Failed to update { item[ 'rel_path' ] } : { e } [/red]" )
329
330 # Delete missing files
331 if to_delete:
332 task = progress.add_task( f "Deleting { len (to_delete) } file(s)..." , total = len (to_delete))
333 for item in to_delete:
334 try :
335 asst.delete_file( file_id = item[ 'remote_file_id' ])
336 deleted_count += 1
337 progress.advance(task)
338 except Exception as e:
339 console.print( f "[red]Failed to delete { item[ 'rel_path' ] } : { e } [/red]" )
340
341 # Final summary
342 console.print()
343 console.print(Panel(
344 f "[green]✓ Sync complete![/green] \n\n "
345 f "Uploaded: { uploaded_count }\n "
346 f "Updated: { updated_count }\n "
347 + ( f "Deleted: { deleted_count }\n " if delete_missing else "" ) +
348 f "Unchanged: { len (unchanged) } " ,
349 title = "Results" ,
350 border_style = "green"
351 ))
352
353 except Exception as e:
354 console.print( f "[red]Error: { e } [/red]" )
355 raise typer.Exit( 1 )
356
357
358 if __name__ == "__main__" :
359 app()