|
@@ -94,12 +94,13 @@ Downloader::
|
|
|
int Downloader::
|
|
int Downloader::
|
|
|
connect_to_server_by_proxy(const string &proxy_name, uint proxy_port,
|
|
connect_to_server_by_proxy(const string &proxy_name, uint proxy_port,
|
|
|
const string &server_name) {
|
|
const string &server_name) {
|
|
|
- if (connect_to_server(proxy_name, proxy_port) != EU_success)
|
|
|
|
|
|
|
+ if (connect_to_server(proxy_name, proxy_port) != EU_success) {
|
|
|
downloader_cat.error()
|
|
downloader_cat.error()
|
|
|
<< "Downloader::connect_to_server_by_proxy() - could not connect to: "
|
|
<< "Downloader::connect_to_server_by_proxy() - could not connect to: "
|
|
|
<< proxy_name << endl;
|
|
<< proxy_name << endl;
|
|
|
return EU_error_abort;
|
|
return EU_error_abort;
|
|
|
-
|
|
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
_proxy_string = "http://";
|
|
_proxy_string = "http://";
|
|
|
_proxy_string += server_name;
|
|
_proxy_string += server_name;
|
|
|
_use_proxy = true;
|
|
_use_proxy = true;
|
|
@@ -115,8 +116,6 @@ connect_to_server_by_proxy(const string &proxy_name, uint proxy_port,
|
|
|
int Downloader::
|
|
int Downloader::
|
|
|
connect_to_server(const string &name, uint port) {
|
|
connect_to_server(const string &name, uint port) {
|
|
|
|
|
|
|
|
- _use_proxy = false;
|
|
|
|
|
-
|
|
|
|
|
#if defined(WIN32)
|
|
#if defined(WIN32)
|
|
|
if (_TCP_stack_initialized == false) {
|
|
if (_TCP_stack_initialized == false) {
|
|
|
WSAData mydata;
|
|
WSAData mydata;
|
|
@@ -295,8 +294,8 @@ fast_receive(int socket, DownloadStatus *status, int rec_size) {
|
|
|
<< handle_socket_error() << endl;
|
|
<< handle_socket_error() << endl;
|
|
|
return get_network_error();
|
|
return get_network_error();
|
|
|
}
|
|
}
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::fast_receive() - recv() requested: " << rec_size
|
|
<< "Downloader::fast_receive() - recv() requested: " << rec_size
|
|
|
<< " got: " << ret << " bytes" << endl;
|
|
<< " got: " << ret << " bytes" << endl;
|
|
|
status->_next_in += ret;
|
|
status->_next_in += ret;
|
|
@@ -506,8 +505,8 @@ run(void) {
|
|
|
|
|
|
|
|
// Recompute the buffer size if necessary
|
|
// Recompute the buffer size if necessary
|
|
|
if (_recompute_buffer == true) {
|
|
if (_recompute_buffer == true) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - Recomputing the buffer" << endl;
|
|
<< "Downloader::run() - Recomputing the buffer" << endl;
|
|
|
|
|
|
|
|
// Flush the current buffer if it holds any data
|
|
// Flush the current buffer if it holds any data
|
|
@@ -535,8 +534,8 @@ run(void) {
|
|
|
} else if (_current_status->_bytes_in_buffer + _receive_size > ((unsigned int)_disk_buffer_size)) {
|
|
} else if (_current_status->_bytes_in_buffer + _receive_size > ((unsigned int)_disk_buffer_size)) {
|
|
|
|
|
|
|
|
// Flush the current buffer if the next request would overflow it
|
|
// Flush the current buffer if the next request would overflow it
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - Flushing buffer" << endl;
|
|
<< "Downloader::run() - Flushing buffer" << endl;
|
|
|
write_ret = write_to_disk(_current_status);
|
|
write_ret = write_to_disk(_current_status);
|
|
|
if (write_ret < 0)
|
|
if (write_ret < 0)
|
|
@@ -550,8 +549,8 @@ run(void) {
|
|
|
if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
|
|
if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
|
|
|
int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
|
|
int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
|
|
|
int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
|
|
int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - fast connection - repeat: " << repeat
|
|
<< "Downloader::run() - fast connection - repeat: " << repeat
|
|
|
<< " remain: " << remain << endl;
|
|
<< " remain: " << remain << endl;
|
|
|
// Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
|
|
// Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
|
|
@@ -568,8 +567,8 @@ run(void) {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
} else { // Handle the normal speed connection case
|
|
} else { // Handle the normal speed connection case
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - normal connection" << endl;
|
|
<< "Downloader::run() - normal connection" << endl;
|
|
|
fret = fast_receive(_socket, _current_status, _receive_size);
|
|
fret = fast_receive(_socket, _current_status, _receive_size);
|
|
|
}
|
|
}
|
|
@@ -584,20 +583,20 @@ run(void) {
|
|
|
if (write_ret < 0)
|
|
if (write_ret < 0)
|
|
|
return write_ret;
|
|
return write_ret;
|
|
|
}
|
|
}
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - Got eof" << endl;
|
|
<< "Downloader::run() - Got eof" << endl;
|
|
|
cleanup();
|
|
cleanup();
|
|
|
return EU_success;
|
|
return EU_success;
|
|
|
} else {
|
|
} else {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - Got 0 bytes" << endl;
|
|
<< "Downloader::run() - Got 0 bytes" << endl;
|
|
|
return ret;
|
|
return ret;
|
|
|
}
|
|
}
|
|
|
} else if (fret == EU_network_no_data) {
|
|
} else if (fret == EU_network_no_data) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run() - No data" << endl;
|
|
<< "Downloader::run() - No data" << endl;
|
|
|
return ret;
|
|
return ret;
|
|
|
} else if (fret < 0) {
|
|
} else if (fret < 0) {
|
|
@@ -629,8 +628,8 @@ run_to_ram(void) {
|
|
|
|
|
|
|
|
// Recompute the buffer size if necessary
|
|
// Recompute the buffer size if necessary
|
|
|
if (_recompute_buffer == true) {
|
|
if (_recompute_buffer == true) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - Recomputing the buffer" << endl;
|
|
<< "Downloader::run_to_ram() - Recomputing the buffer" << endl;
|
|
|
|
|
|
|
|
// Flush the current buffer if it holds any data
|
|
// Flush the current buffer if it holds any data
|
|
@@ -658,8 +657,8 @@ run_to_ram(void) {
|
|
|
((unsigned int)_disk_buffer_size)) {
|
|
((unsigned int)_disk_buffer_size)) {
|
|
|
|
|
|
|
|
// Flush the current buffer if the next request would overflow it
|
|
// Flush the current buffer if the next request would overflow it
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - Flushing buffer" << endl;
|
|
<< "Downloader::run_to_ram() - Flushing buffer" << endl;
|
|
|
write_ret = write_to_ram(_current_status);
|
|
write_ret = write_to_ram(_current_status);
|
|
|
if (write_ret < 0)
|
|
if (write_ret < 0)
|
|
@@ -673,8 +672,8 @@ run_to_ram(void) {
|
|
|
if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
|
|
if (_receive_size > (ulong)MAX_RECEIVE_BYTES) {
|
|
|
int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
|
|
int repeat = (int)(_receive_size / MAX_RECEIVE_BYTES);
|
|
|
int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
|
|
int remain = (int)fmod((double)_receive_size, (double)MAX_RECEIVE_BYTES);
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - fast connection - repeat: " << repeat
|
|
<< "Downloader::run_to_ram() - fast connection - repeat: " << repeat
|
|
|
<< " remain: " << remain << endl;
|
|
<< " remain: " << remain << endl;
|
|
|
// Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
|
|
// Make multiple requests at once but do not exceed MAX_RECEIVE_BYTES
|
|
@@ -691,8 +690,8 @@ run_to_ram(void) {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
} else { // Handle the normal speed connection case
|
|
} else { // Handle the normal speed connection case
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - normal connection" << endl;
|
|
<< "Downloader::run_to_ram() - normal connection" << endl;
|
|
|
fret = fast_receive(_socket, _current_status, _receive_size);
|
|
fret = fast_receive(_socket, _current_status, _receive_size);
|
|
|
}
|
|
}
|
|
@@ -707,20 +706,20 @@ run_to_ram(void) {
|
|
|
if (write_ret < 0)
|
|
if (write_ret < 0)
|
|
|
return write_ret;
|
|
return write_ret;
|
|
|
}
|
|
}
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - Got eof" << endl;
|
|
<< "Downloader::run_to_ram() - Got eof" << endl;
|
|
|
cleanup();
|
|
cleanup();
|
|
|
return EU_success;
|
|
return EU_success;
|
|
|
} else {
|
|
} else {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - Got 0 bytes" << endl;
|
|
<< "Downloader::run_to_ram() - Got 0 bytes" << endl;
|
|
|
return ret;
|
|
return ret;
|
|
|
}
|
|
}
|
|
|
} else if (fret == EU_network_no_data) {
|
|
} else if (fret == EU_network_no_data) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::run_to_ram() - No data" << endl;
|
|
<< "Downloader::run_to_ram() - No data" << endl;
|
|
|
return ret;
|
|
return ret;
|
|
|
} else if (fret < 0) {
|
|
} else if (fret < 0) {
|
|
@@ -838,8 +837,8 @@ parse_header(DownloadStatus *status) {
|
|
|
status->_first_line_complete = true;
|
|
status->_first_line_complete = true;
|
|
|
int parse_ret = parse_http_response(component);
|
|
int parse_ret = parse_http_response(component);
|
|
|
if (parse_ret == EU_success) {
|
|
if (parse_ret == EU_success) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::parse_header() - Header is valid: "
|
|
<< "Downloader::parse_header() - Header is valid: "
|
|
|
<< component << endl;
|
|
<< component << endl;
|
|
|
status->_header_is_valid = true;
|
|
status->_header_is_valid = true;
|
|
@@ -886,8 +885,8 @@ parse_header(DownloadStatus *status) {
|
|
|
<< "Location directive" << endl;
|
|
<< "Location directive" << endl;
|
|
|
return EU_error_abort;
|
|
return EU_error_abort;
|
|
|
}
|
|
}
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::parse_header() - Header is complete" << endl;
|
|
<< "Downloader::parse_header() - Header is complete" << endl;
|
|
|
status->_header_is_complete = true;
|
|
status->_header_is_complete = true;
|
|
|
|
|
|
|
@@ -896,8 +895,8 @@ parse_header(DownloadStatus *status) {
|
|
|
status->_start += header_length;
|
|
status->_start += header_length;
|
|
|
status->_bytes_in_buffer -= header_length;
|
|
status->_bytes_in_buffer -= header_length;
|
|
|
|
|
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::parse_header() - Stripping out header of size: "
|
|
<< "Downloader::parse_header() - Stripping out header of size: "
|
|
|
<< header_length << endl;
|
|
<< header_length << endl;
|
|
|
|
|
|
|
@@ -946,8 +945,8 @@ write_to_disk(DownloadStatus *status) {
|
|
|
|
|
|
|
|
// Write what we have so far to disk
|
|
// Write what we have so far to disk
|
|
|
if (status->_bytes_in_buffer > 0) {
|
|
if (status->_bytes_in_buffer > 0) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::write_to_disk() - Writing "
|
|
<< "Downloader::write_to_disk() - Writing "
|
|
|
<< status->_bytes_in_buffer << " to disk" << endl;
|
|
<< status->_bytes_in_buffer << " to disk" << endl;
|
|
|
|
|
|
|
@@ -990,8 +989,8 @@ write_to_ram(DownloadStatus *status) {
|
|
|
|
|
|
|
|
// Write what we have so far to memory
|
|
// Write what we have so far to memory
|
|
|
if (status->_bytes_in_buffer > 0) {
|
|
if (status->_bytes_in_buffer > 0) {
|
|
|
- if (downloader_cat.is_debug())
|
|
|
|
|
- downloader_cat.debug()
|
|
|
|
|
|
|
+ if (downloader_cat.is_spam())
|
|
|
|
|
+ downloader_cat.spam()
|
|
|
<< "Downloader::write_to_ram() - Writing "
|
|
<< "Downloader::write_to_ram() - Writing "
|
|
|
<< status->_bytes_in_buffer << " to memory" << endl;
|
|
<< status->_bytes_in_buffer << " to memory" << endl;
|
|
|
|
|
|