Setting the file. One moment.
Find Models By Paper · Huggingface Tool Builder · huggingface/skills · Skills Docs
Back to the top of the page references/find_models_by_paper.sh
references/ find_models_by_paper.sh
Shell · 230 lines · 8 KB
=
'\033[0m'
# No Color
15
16 # Help function
17 show_help () {
18 echo -e "${ BLUE }Find models associated with papers on Hugging Face${ NC }"
19 echo ""
20 echo -e "${ YELLOW }Usage:${ NC }"
21 echo " $0 [OPTIONS] [search_term|arXiv_id]"
22 echo ""
23 echo -e "${ YELLOW }Options:${ NC }"
24 echo " --help Show this help message"
25 echo " --token Use HF_TOKEN environment variable (if set)"
26 echo ""
27 echo -e "${ YELLOW }Environment:${ NC }"
28 echo " HF_TOKEN Optional: Hugging Face token for private/gated models"
29 echo ""
30 echo -e "${ YELLOW }Examples:${ NC }"
31 echo " $0 1910.01108 # Search by arXiv ID"
32 echo " $0 distilbert # Search by model name"
33 echo " $0 transformer # Search by keyword"
34 echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication"
35 echo ""
36 echo -e "${ YELLOW }Description:${ NC }"
37 echo "This script finds Hugging Face models that are associated with research papers."
38 echo "It searches for models that have arXiv IDs in their tags or mentions papers in their metadata."
39 echo ""
40 echo -e "${ YELLOW }Notes:${ NC }"
41 echo "• HF_TOKEN is optional for public models"
42 echo "• Use HF_TOKEN for private repositories or gated models"
43 echo "• HF_TOKEN enables higher rate limits for heavy usage"
44 }
45
46 # Parse arguments
47 USE_TOKEN = false
48 POSITIONAL_ARGS = ()
49
50 while [[ $# -gt 0 ]]; do
51 case $1 in
52 --help )
53 show_help
54 exit 0
55 ;;
56 --token )
57 USE_TOKEN = true
58 shift
59 ;;
60 - * )
61 echo -e "${ RED }Unknown option: $1 ${ NC }"
62 show_help
63 exit 1
64 ;;
65 *)
66 POSITIONAL_ARGS += ( " $1 " )
67 shift
68 ;;
69 esac
70 done
71
72 set -- "${ POSITIONAL_ARGS [ @ ]}"
73
74 if [[ $# -eq 0 ]]; then
75 echo -e "${ RED }Error: Please provide a search term or arXiv ID${ NC }"
76 echo -e "Use ${ YELLOW } $0 --help${ NC } for usage information"
77 exit 1
78 fi
79
80 SEARCH_TERM = " $1 "
81
82 # Set up authentication header if HF_TOKEN is available
83 if [[ -n " $HF_TOKEN " ]] && [[ " $USE_TOKEN " == true || -n " $HF_TOKEN " ]]; then
84 AUTH_HEADER = "-H \" Authorization: Bearer $HF_TOKEN \" "
85 echo -e "${ BLUE }Using HF_TOKEN for authentication${ NC }"
86 else
87 AUTH_HEADER = ""
88 if [[ -n " $HF_TOKEN " ]]; then
89 echo -e "${ YELLOW }HF_TOKEN found but not using it (add --token flag to use)${ NC }"
90 fi
91 fi
92
93 # Check if the input looks like an arXiv ID (format: YYYY.NNNNN or YYYY.NNNNNNN)
94 if [[ " $SEARCH_TERM " =~ ^[0-9]{ 4 } \. [0-9]{4,7}$ ]]; then
95 echo -e "${ BLUE }Searching for models associated with arXiv paper: $SEARCH_TERM ${ NC }"
96 SEARCH_QUERY = "arxiv%3A $SEARCH_TERM "
97 IS_ARXIV_SEARCH = true
98 else
99 echo -e "${ BLUE }Searching for models related to: $SEARCH_TERM ${ NC }"
100 SEARCH_QUERY = " $SEARCH_TERM "
101 IS_ARXIV_SEARCH = false
102 fi
103
104 # Function to extract arXiv IDs from tags
105 extract_arxiv_ids () {
106 local tags = " $1 "
107 echo " $tags " | jq -r '.[] | select(. | startswith("arxiv:")) | split(":")[1]' 2> /dev/null || true
108 }
109
110 # Function to get paper title from arXiv ID
111 get_paper_title () {
112 local arxiv_id = " $1 "
113 # Try to get paper title from Hugging Face tags if available
114 # This is a simplified approach - in practice, you might want to call arXiv API
115 echo "Paper Title (arXiv: $arxiv_id )"
116 }
117
118 # Search for models
119 API_URL = "https://huggingface.co/api/models"
120 echo -e "${ YELLOW }Searching Hugging Face API...${ NC }"
121
122 # Build curl command with authentication if available
123 CURL_CMD = "curl -s $AUTH_HEADER \" $API_URL ?search= $SEARCH_QUERY &limit=50 \" "
124 echo -e "${ BLUE }API Query: $API_URL ?search= $SEARCH_QUERY &limit=50${ NC }"
125
126 # Execute the API call
127 if [[ -n " $HF_TOKEN " ]]; then
128 RESPONSE = $( curl -s -H "Authorization: Bearer $HF_TOKEN " " $API_URL ?search= $SEARCH_QUERY &limit=50" || true )
129 else
130 RESPONSE = $( curl -s " $API_URL ?search= $SEARCH_QUERY &limit=50" || true )
131 fi
132
133 # Check if we got a valid response
134 if [[ -z " $RESPONSE " ]] || [[ " $RESPONSE " == "[]" ]]; then
135 echo -e "${ RED }No models found for search term: $SEARCH_TERM ${ NC }"
136
137 # If arXiv search failed, try without arxiv: prefix
138 if [[ " $IS_ARXIV_SEARCH " == true ]]; then
139 echo -e "${ YELLOW }Trying broader search without arxiv: prefix...${ NC }"
140 SEARCH_QUERY = " $SEARCH_TERM "
141 IS_ARXIV_SEARCH = false
142
143 if [[ -n " $HF_TOKEN " ]]; then
144 RESPONSE = $( curl -s -H "Authorization: Bearer $HF_TOKEN " " $API_URL ?search= $SEARCH_QUERY &limit=50" || true )
145 else
146 RESPONSE = $( curl -s " $API_URL ?search= $SEARCH_QUERY &limit=50" || true )
147 fi
148
149 if [[ -z " $RESPONSE " ]] || [[ " $RESPONSE " == "[]" ]]; then
150 echo -e "${ RED }Still no results found. Try a different search term.${ NC }"
151 exit 1
152 fi
153 else
154 exit 1
155 fi
156 fi
157
158 # Process the results
159 echo -e "${ GREEN }Found models! Processing results...${ NC }"
160
161 # Use jq to process the JSON response and find models with paper associations
162 MODELS_WITH_PAPERS = $( echo " $RESPONSE " | jq -r '
163 .[] |
164 select(.id != null) |
165 {
166 id: .id,
167 arxiv_tags: [.tags[] | select(. | startswith("arxiv:"))] | join("; "),
168 downloads: (.downloads // 0),
169 likes: (.likes // 0),
170 task: (.pipeline_tag // "unknown"),
171 library: (.library_name // "unknown")
172 }
173 | @base64' 2> /dev/null || true )
174
175 # Count total results
176 TOTAL_MODELS = $( echo " $RESPONSE " | jq 'length' 2> /dev/null || echo "0" )
177 MODELS_WITH_PAPERS_COUNT = $( echo " $MODELS_WITH_PAPERS " | wc -l )
178
179 echo -e "${ BLUE }Results Summary:${ NC }"
180 echo -e " Total models found: $TOTAL_MODELS "
181 echo -e " Models with paper associations: $MODELS_WITH_PAPERS_COUNT "
182 echo ""
183
184 if [[ -z " $MODELS_WITH_PAPERS " ]]; then
185 # Show all models even if no paper associations found
186 echo -e "${ YELLOW }No explicit paper associations found. Showing all matching models:${ NC }"
187 echo " $RESPONSE " | jq -r '
188 .[] |
189 select(.id != null) |
190 "📦 \(.id)
191 Task: \(.pipeline_tag // "unknown")
192 Downloads: \(.downloads // 0)
193 Likes: \(.likes // 0)
194 Library: \(.library_name // "unknown")
195 ---"
196 ' 2> /dev/null || echo "Failed to parse response"
197 else
198 # Show models with paper associations
199 echo -e "${ GREEN }Models with paper associations:${ NC }"
200 echo " $MODELS_WITH_PAPERS " | while read -r model_data ; do
201 if [[ -n " $model_data " ]]; then
202 # Decode base64 and show formatted
203 echo " $model_data " | base64 -d | jq -r '
204 "📄 \(.id)
205 arXiv: \(.arxiv_tags)
206 Task: \(.task)
207 Downloads: \(.downloads)
208 Likes: \(.likes)
209 Library: \(.library)
210 ---"
211 ' 2> /dev/null || echo "Failed to parse model data"
212 fi
213 done
214 fi
215
216 # Additional search tips
217 echo ""
218 echo -e "${ BLUE }Search Tips:${ NC }"
219 echo "• Try searching with the full arXiv ID (e.g., 1910.01108)"
220 echo "• Try searching with the paper title keywords"
221 echo "• Try searching with the model name"
222 echo "• Use HF_TOKEN for private models or higher rate limits"
223 echo ""
224 echo -e "${ BLUE }Examples to try:${ NC }"
225 echo " $0 1910.01108 # DistilBERT paper"
226 echo " $0 1810.04805 # BERT paper"
227 echo " $0 1706.03762 # Attention is All You Need paper"
228 echo " $0 roberta # RoBERTa models"
229 echo " $0 transformer # Transformer models"
230 echo " HF_TOKEN=your_token $0 1910.01108 # Use authentication"