|
|
@@ -550,7 +550,7 @@ def add(urls: Union[str, List[str]],
|
|
|
if new_links and depth == 1:
|
|
|
log_crawl_started(new_links)
|
|
|
for new_link in new_links:
|
|
|
- downloaded_file = save_file_as_source(new_link.url, filename='{ts}-crawl-{basename}.txt', out_dir=out_dir)
|
|
|
+ downloaded_file = save_file_as_source(new_link.url, filename=f'{new_link.timestamp}-crawl-{new_link.domain}.txt', out_dir=out_dir)
|
|
|
new_links_depth += parse_links_from_source(downloaded_file)
|
|
|
|
|
|
imported_links = new_links + new_links_depth
|