-
Notifications
You must be signed in to change notification settings - Fork 9
Expand file tree
/
Copy pathmapScraperX.py
More file actions
139 lines (112 loc) · 5.09 KB
/
Copy pathmapScraperX.py
File metadata and controls
139 lines (112 loc) · 5.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import sys
import os
import argparse
import time
from tqdm import tqdm
import mapScraper.places_crawler as crawler
def read_queries_from_file(file_path):
"""Read queries from a text file"""
queries = []
try:
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
query = line.strip()
if query and not query.startswith('#'):
queries.append(query)
return queries
except FileNotFoundError:
print(f"Error: Could not find the file {file_path}")
return []
except Exception as e:
print(f"Error reading the file {file_path}: {e}")
return []
def process_single_query(query, lang, country, limit, mode):
"""Process a single query"""
print(f"Processing query: '{query}' (mode: {mode})")
results = crawler.search(query, lang, country, limit, mode)
print(f"Found {len(results)} results for '{query}'")
return results
def process_multiple_queries(queries, lang, country, limit_per_query, max_concurrent, mode):
"""Process multiple queries concurrently"""
total_queries = len(queries)
print(f"Processing {total_queries} queries concurrently (max {max_concurrent} at a time)...")
print(f"Scraping mode: {mode}")
start_time = time.time()
# Use async search for multiple queries
results = crawler.search_multiple_sync(
queries,
lang,
country,
limit_per_query,
max_concurrent,
mode
)
elapsed_time = time.time() - start_time
print(f"\nCompleted in {elapsed_time:.2f} seconds")
print(f"Average time per query: {elapsed_time/total_queries:.2f} seconds")
return results
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Scrape Google Maps for local services with concurrent processing.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Scraping Modes:
fast - Main search only, no phone fallback (fastest)
standard - Main search + phone fallback (recommended)
complete - Main search + phone + address fallback (most thorough)
Examples:
Single query:
python mapScraperX.py "restaurants in Miami" --limit 50
With specific mode:
python mapScraperX.py "gym in vandavasi" --mode complete
Multiple queries (concurrent processing by default):
python mapScraperX.py --queries-file query_example.txt
Adjust concurrency level:
python mapScraperX.py --queries-file query_example.txt --concurrent 5
"""
)
query_group = parser.add_mutually_exclusive_group(required=True)
query_group.add_argument('query', nargs='?', type=str, help='The search query.')
query_group.add_argument('--queries-file', type=str, help='Path to a text file containing one query per line.')
parser.add_argument('--lang', type=str, default='en', help='Language code (e.g., en, es, fr). Default: en')
parser.add_argument('--country', type=str, default='us', help='Country code (e.g., us, es, fr). Default: us')
parser.add_argument('--limit', type=int, default=0, help='Max results (total for single query, per query for file mode). Default: no limit')
parser.add_argument('--output-file', type=str, default='data/output.csv', help='Output CSV file path. Default: data/output.csv')
parser.add_argument('--concurrent', type=int, default=3, help='Max concurrent queries (default: 3, recommended: 3-5)')
parser.add_argument('--mode', type=str, choices=['fast', 'standard', 'complete'], default='standard',
help='Scraping mode: fast (no phone fallback), standard (skip fallback if any phone exists), complete (fallback always). Default: standard')
args = parser.parse_args()
output_dir = os.path.dirname(args.output_file)
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
if args.query:
print("Mode: Single query")
results = process_single_query(args.query, args.lang, args.country, args.limit, args.mode)
elif args.queries_file:
print(f"Mode: Multiple queries from file ({args.queries_file})")
queries = read_queries_from_file(args.queries_file)
if not queries:
print("No valid queries found in the file.")
sys.exit(1)
print(f"Concurrent processing enabled: {args.concurrent} queries at a time")
print(f"Scraping mode: {args.mode}")
if args.limit:
print(f"Limit per query: {args.limit}")
results = process_multiple_queries(
queries,
args.lang,
args.country,
args.limit,
max_concurrent=args.concurrent,
mode=args.mode
)
if results:
crawler.save_to_csv(results, args.output_file)
print(f"\n{'='*50}")
print(f"Final Summary:")
print(f" Total results: {len(results)}")
print(f" Scraping mode: {args.mode}")
print(f" File saved to: {args.output_file}")
print(f"{'='*50}")
else:
print("No results found.")