/* httpsync.c version 3.00 Copyright 2001 Forrest J. Cavalier III See http://www.mibsoftware.com/httpsync/ for documentation and original copies of this software. Contact Mib Software (mibsoft@mibsoftware.com) to discuss custom and enhanced versions of this and other software. We would appreciate that you notify us of defect discoveries and enhancements so that others can benefit from work on this software. - - - - - - - - - - - - - License - - - - - - - - - - - - - - - - - Copyright (c) 2001, Forrest J. Cavalier III, doing business as Mib Software Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /*****************************************************************/ /* CHANGES Versopm 3.00 [imp] SSL features Version 2.01 [imp] HTTP authorization Version 2.00 [imp] Feature enhanced for automated interface. Version 1.14 [cri] HTTPAccess_ReadHead() would hang when buffer read so far ended \n but not \n\n [def,opt]".." in comments (not just file names) would cause termination with error. Using memstr() instead of strstr() also saves CPU time in long file lists. [def] Files ending with comments would cause unrecognized line errors. Version 1.13 [mai] Syntax clean up to keep some compilers happy. Dropped register keyword in ndays(). Added casts for a printf argument based on mode_t, and off_t. Version 1.12 [cri] HTTP/1.0 bodies received after a delay would write a zero length buffer through PackListAppend(). Added test for empty buffer. Version 1.11 [cri] Modified URLs requested to eliminate '/./' within HTTP requests. Netscape Enterprise server, for one, rejects them. Version 1.10 [mai] Rewrote HTTP handling, other changes for DDJ article. Version 1.03 GENMODMASK used when generating mode values. Preserve R lines when generating lists. chmod to allow distribution of mode 04xx files. Need to chmod after transfer. Incorporate umask into chmod call. Version 1.02: When could not open destination file for writing, close socket before continuing with next file. Obsolete processing: rmdir() and unlink() return codes checked and warning printed. MODMASK allows x bit to be ignored on MS-DOS file systems. Version 1.01: Features added. Obsolete file removal, file modes, HTTP/1.1 transfers. Version 1.00: Introduced. */ /*****************************************************************/ /* Setup HTTPSYNC compilation environment. Handle various operating system differences. */ #ifdef NeXT #define NO_INC_MALLOC #endif #ifndef NO_INC_MALLOC #include #endif #include #include #include #include #include #include #include #include #include #include /**********************************************************/ /* WinSock and BSD-style sockets portability */ /* Macros and conditionals for portable sockets code * 1. Write code using readsocket, writesocket, closesocket, * INVALID_SOCKET, SOCKET_ERROR, SOCKET, and INADDR_NONE * 2. Always use SocketStartup() and SocketCleanup() * 3. Conditional includes and definitions for those macros * allow operation under Windows and BSD-style sockets. */ #ifdef WIN32 /* Windows systems */ #include #include #define readsocket(a,b,c) recv(a,b,c,0) #define writesocket(a,b,c) send(a,b,c,0) /* closesocket() does not need a macro. * INVALID_SOCKET, SOCKET_ERROR, SOCKET, and INADDR_NONE * are already defined in winsock.h */ WSADATA libmibWSAdata; #define SocketStartup() \ if (WSAStartup(0x101,&libmibWSAdata)) exit(-1) #define SocketCleanup() WSACleanup() #else /* Unix-style systems */ #include #include #include #include #define readsocket read #define writesocket write #define closesocket close #define SocketStartup() #define SocketCleanup() #define INVALID_SOCKET -1 #define SOCKET_ERROR -1 #define SOCKET int /* define INADDR_NONE if not already */ #ifndef INADDR_NONE #define INADDR_NONE ((unsigned long) -1) #endif #endif /*****************************/ /* Additional macros and environment for portable * HTTPsync code */ #ifdef WIN32 #include #include #define TDIFFLIMIT 2 /* MS-DOS file timestamp resolution */ #define MODMASK 0600 /* When comparing, don't check Execute, group, or other bits */ #define GENMODMASK 0644 /* When generating, use only rw-r-r bits */ #define strncasecmp strnicmp #define portable_mkdir(d,perm) mkdir(d) #define FOPEN_WRITE_BINARY "wb" #ifdef __MINGW32__ /* Mingw32/egcs on Win32.Thx to Ron Aaron */ #include #endif #else /* Unix-like systems */ #include #include #include #include #define portable_mkdir(d,perm) mkdir(d,perm) #define FOPEN_WRITE_BINARY "w" #define TDIFFLIMIT 1 #define MODMASK 0700 /* Check only that owner modes match */ #define GENMODMASK 0755 /* For generating packing lists. */ #endif #include #ifdef NeXT #include #include #define utimbuf myutimbuf struct myutimbuf { time_t actime; time_t modtime; }; #define strdup NXCopyStringBuffer #endif /*****************************************************************/ /* -- End setup of compilation environment. There should be no O/S * dependent conditionals below this line. */ /*****************************************************************/ #ifdef USE_SSLEAY #include "rsa.h" /* SSLeay stuff */ #include "crypto.h" #include "x509.h" #include "pem.h" #include "ssl.h" #include "err.h" #define do_SSL_CTX_new() SSL_CTX_new() #define ENABLE_SSL #endif #ifdef USE_OPENSSL #include "openssl/rsa.h" #include "openssl/crypto.h" #include "openssl/x509.h" #include "openssl/pem.h" #include "openssl/ssl.h" #include "openssl/err.h" #define do_SSL_CTX_new() SSL_CTX_new(SSLv23_client_method()) #define ENABLE_SSL #endif #ifndef ENABLE_SSL #define SSL_CTX void #define SSL void #define SSL_free(a) #define SSL_CTX_free(a) #define do_SSL_CTX_new() #define SSL_load_error_strings() 0 #define SSL_read(a,b,c) 0 #define SSL_write(a,b,c) 0 #endif /* NOTE: At runtime, must be able to find crypt32.dll and ssl32.dll from SSLeay. */ /*****************************************************************/ /* Predeclarations. These functions are defined in this file after they are referenced. */ int ssgmtime(const char *sptr,time_t *time); int countbl(const char *pStart); /* count consecutive ' ' and '\t' */ int counttoch(const char *pStart,char ch); /* count to \0 or ch */ char *memstr(const char *pBlock,const char *pSubstring,int cbBlock); int counttowh(const char *pStart); /* count to \0 ' ' '\t' '\r' or '\n' */ char *astrn0cpy(char **asz,const char *src,int len); char *astrn0cat(char **asz,const char *src,int len); char *URIunescape(char *asz); /* converts %xx */ time_t tm_to_time (struct tm *tp); #define PACKLISTVER 200 /* 100 does not support 'R', 'O' tags 101 still the default format 200 supports % escaped file names */ int USEVER = PACKLISTVER; int bVERBOSE = 0; int umaskval; /**************************************************************/ int bAUTOMATED = 0; /* -a option */ char *aszWorkDest = 0; /* -t option */ int cbTempdir; long cbAllFiles = 0; long cAllFiles = 0; long cbCompleted = 0; long cFileCompleted = 0; /**************************************************************/ /**************************************************************/ /* Arguments which are processed from command line */ char *pszURI; /**************************************************************/ /* HTTPaccess implements a reusable method of making multiple requests to an HTTP server. Tries HTTP/1.1 and falls back to HTTP/1.0 after a number of failed HTTP/1.1 attempts. For example use, see mainHTTPSYNC(), basically call HTTPaccess_Initialize() to initialize a structure with the host names of servers, then make multiple requests through HTTPaccess_Retrieve() For advanced usage, customize HTTPaccess_Retrieve() or HTTPaccess_SendRequest() */ typedef char EXPLAIN; /* For return values that are error explanations including a decimal ASCII number at position 2. Success is returned as 0 (NULL). */ struct HTTPaccess_s { SOCKET s; int bPERSIST; /* already a connection open */ int bHTTP1_1; /* Attempt connection as HTTP/1.1 */ struct sockaddr_in saddr; int bSKIPDNS; /* Set nonzero when saddr is filled in, the lookup is done once */ const char *pszHost; int nPort; char *pszAuthUser; char *pszAuthPass; const char *pszProxy; int nProxyPort; long cbContent; /* Content length, or one of the following */ #define HTTPaccess_CHUNKED -1 #define HTTPaccess_UNTILCLOSE -2 int cbInbuf; char *buf; int cbBuf; int cAttempt; SSL_CTX* ctx; SSL* ssl; }; void HTTPaccess_Initialize(struct HTTPaccess_s *pHTA, char * buf,int cbBuf) { /* buf must be large enough to hold all response headers, or error will be returned. */ pHTA->bHTTP1_1 = 1; /* default. Falls back automatically. */ pHTA->bSKIPDNS = 0; pHTA->pszProxy = 0; pHTA->pszHost = 0; pHTA->bPERSIST = 0; pHTA->nPort = 80; /* default */ pHTA->buf = buf; pHTA->cbBuf = cbBuf; pHTA->ssl = 0; pHTA->pszAuthUser = 0; pHTA->pszAuthPass = 0; } /* HTTPaccess_Initialize */ const char *HTTPaccess_ParseURL(struct HTTPaccess_s *pHTA,const char *pszURL,const char *pszProxy) { char *ptr; pHTA->nPort = 80; /* http default */ if (!strncasecmp(pszURL,"http://",7)) { pHTA->pszHost = pszURL+7; } else if (!strncasecmp(pszURL,"https://",8)) { pHTA->pszHost = pszURL+8; pHTA->ssl = (struct ssl_st *) 1; /* Will be replaced at OpenConn */ pHTA->nPort = 443; } else { return "E-1-Invalid URL syntax"; } /* Strip out the host */ if (!(ptr = strchr(pHTA->pszHost,'/'))) { return "E-1-Invalid URL syntax"; } *ptr = '\0'; pszURI = ptr+1; if ((ptr = strchr(pHTA->pszHost,':'))) { *ptr = 0; pHTA->nPort = atoi(ptr+1); } if (pszProxy) { pHTA->nProxyPort = 443; if ((ptr = strchr(pszProxy,':'))) { *ptr = 0; pHTA->nProxyPort = atoi(ptr+1); } } return 0; } /* HTTPaccess_ParseURL */ EXPLAIN *HTTPaccess_OpenConn(struct HTTPaccess_s *pHTA) { /* A "helper" function called by HTTPaccess_SendRequest */ /* Returns error explanation */ /* Lookup host name */ const char *pszHost = pHTA->pszHost; int nPort = pHTA->nPort; struct hostent *phostent; if (pHTA->bPERSIST) { /* Already a HTTP/1.1 persistent connection open */ return 0; } if (pHTA->pszProxy) { pszHost = pHTA->pszProxy; nPort = pHTA->nProxyPort; } if (!pHTA->bSKIPDNS) { phostent = gethostbyname(pszHost); pHTA->saddr.sin_family = PF_INET; pHTA->saddr.sin_port = htons(nPort); if (!phostent) { pHTA->saddr.sin_addr.s_addr = inet_addr(pszHost); if (pHTA->saddr.sin_addr.s_addr == INADDR_NONE) { if (bVERBOSE) { fprintf(stderr,"Could not resolve hostname '%s'\n",pszHost); } return "E-3-HTTPaccess_OpenConn: could not resolve name"; } } else { pHTA->saddr.sin_addr.s_addr = *((u_long *) phostent->h_addr); } } pHTA->s = socket(PF_INET,SOCK_STREAM,0); if (pHTA->s == INVALID_SOCKET) { return "E-1-HTTPaccess_OpenConn: could not get socket"; } /* And connect */ pHTA->bSKIPDNS = 0; if (bVERBOSE) { printf("[connect]"); } if (connect(pHTA->s, (struct sockaddr *) &(pHTA->saddr), sizeof(pHTA->saddr)) == SOCKET_ERROR) { closesocket(pHTA->s); if (pHTA->ssl && (pHTA->ssl != (void *) 1)) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } return "E-2-HTTPaccess_OpenConn: could not connect"; } pHTA->bSKIPDNS = 1; #ifdef ENABLE_SSL if (pHTA->ssl) { int err; if (pHTA->pszProxy) { int i; int imax = strlen(pHTA->pszHost)+1024 ; /* Use the CONNECT method, as documented at http://developer.netscape.com/docs/manuals/proxy/ProxyUnx/SSL-TUNL.HTM */ char *buf = malloc(imax); if (!buf) { return "E-6-HTTPaccess_OpenConn: Could not malloc"; } sprintf(buf,"CONNECT %s:%d HTTP/1.0\r\n\r\n",pHTA->pszHost,pHTA->nPort); writesocket(pHTA->s,buf,strlen(buf)); if (bVERBOSE) { fprintf(stderr,"%s",buf); } /* Read until get a blank line or socket close */ i = 0; while(i < imax-1) { if (readsocket(pHTA->s,buf+i,1) != 1) { break; } i++; if ((i >= 4) && !strncmp(buf+i-4,"\r\n\r\n",4)) { break; } if ((i >= 2) && !strncmp(buf+i-2,"\n\n",4)) { break; } } buf[i] = '\0'; if (bVERBOSE) { fprintf(stderr,"%s",buf); } i = 0; while(buf[i] > ' ') { i++; } if (!buf[i]) { free(buf); return "E-8-HTTPaccess_OpenConn: No response from proxy"; } if (buf[i+1] != '2') { if (bVERBOSE) { while(1) { int cnt; cnt = readsocket(pHTA->s,buf,sizeof(buf)); if (cnt <= 0) { break; } fwrite(buf,1,cnt,stderr); } } free(buf); return "E-8-HTTPaccess_OpenConn: CONNECT method did not return status 2xx"; } free(buf); } pHTA->ctx = do_SSL_CTX_new (); if (!pHTA->ctx) { if (bVERBOSE) { char buf[256]; /* ERR_error_string_n(ERR_get_error(),buf,sizeof(buf)); */ ERR_error_string(ERR_get_error(),buf); fprintf(stderr,"%s\n",buf); } return "E-4-HTTPaccess_OpenConn: SSL setup failed"; } pHTA->ssl = SSL_new (pHTA->ctx); if (!pHTA->ctx) { return "E-5-HTTPaccess_OpenConn: SSL setup failed"; } SSL_set_fd (pHTA->ssl, pHTA->s); if (bVERBOSE) { printf("Trying SSL_connect"); } err = SSL_connect (pHTA->ssl); if (err==-1) { return "E-3-HTTPaccess_OpenConn: SSL negotiation failed"; } if (bVERBOSE) { X509* server_cert; char* str; char buf[256]; /* Following two steps are optional and not required for data exchange to be successful. */ /* Get the cipher - opt */ printf ("SSL connection using %s\n", SSL_get_cipher (pHTA->ssl)); /* Get server's certificate (note: beware of dynamic allocation) - opt */ server_cert = SSL_get_peer_certificate (pHTA->ssl); /* DEBUG: We should do certificate verification */ printf ("Server certificate:\n"); #ifdef USE_SSLEAY str = X509_NAME_oneline (X509_get_subject_name (server_cert)); printf ("\t unverified subject: %s\n", str); Free (str); str = X509_NAME_oneline (X509_get_issuer_name (server_cert)); printf ("\t unverified issuer: %s\n", str); Free (str); #endif #ifdef USE_OPENSSL X509_NAME_oneline (X509_get_subject_name (server_cert),buf,sizeof(buf)); printf ("\t unverified subject: %s\n", buf); str = X509_NAME_oneline (X509_get_issuer_name (server_cert),buf,sizeof(buf)); printf ("\t unverified issuer: %s\n", buf); #endif X509_free (server_cert); } } #endif return 0; } /* HTTPaccess_OpenConn */ static char *vector = { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"}; int ToBase64(char *wptr, /* Output buffer. Must have room for 4 characters. */ long *paccum, char ch) { /* paccum should be initialized to 1 or 0 on the first call, then just keep calling, At end of block, caller must pad with '=' so that the length is a multiple of 3. Returns the number of characters written at wptr. */ if (!*paccum) { *paccum = 1; } *paccum <<= 8; *paccum |= ch & 0xff; if (*paccum & 0x01000000) { wptr[3] = vector[*paccum&0x3f]; *paccum >>= 6; wptr[2] = vector[*paccum&0x3f]; *paccum >>= 6; wptr[1] = vector[*paccum&0x3f]; *paccum >>= 6; wptr[0] = vector[*paccum&0x3f]; *paccum >>= 6; return 4; } return 0; } EXPLAIN *HTTPaccess_GenRequest(struct HTTPaccess_s *pHTA, const char *pszURI) {/* Generate headers for an HTTP 1.0 or 1.1 request. NOTE: If HTTP/1.1 the caller must append a Content-length header. See HTTPaccess_Retrieve() for an example */ const char *ptr; char *wptr; long accum; if (strlen(pszURI)+strlen(pHTA->pszHost)*2+100+strlen(pHTA->pszAuthUser ? pHTA->pszAuthUser :"")*2+strlen(pHTA->pszAuthPass ? pHTA->pszAuthPass : "")*2 > (unsigned) pHTA->cbBuf) { return "E-1-HTTPaccess_GenRequest buffer too small"; } if (pHTA->pszProxy && !pHTA->ssl) { if (pszURI[0] == '/') { sprintf(pHTA->buf,"GET http://%s%s", pHTA->pszHost,pszURI); } else { sprintf(pHTA->buf,"GET http://%s/%s", pHTA->pszHost,pszURI); } } else { if (pszURI[0] == '/') { sprintf(pHTA->buf,"GET %s",pszURI); } else { sprintf(pHTA->buf,"GET /%s",pszURI); } } if (pHTA->bHTTP1_1) { strcat(pHTA->buf," HTTP/1.1\r\n"); strcat(pHTA->buf,"Host: "); strcat(pHTA->buf,pHTA->pszHost); strcat(pHTA->buf,"\r\n"); } else { strcat(pHTA->buf," HTTP/1.0\r\n"); } if (pHTA->pszAuthUser) { strcat(pHTA->buf,"Authorization: Basic "); /* Make sure not overrun pHTA->buf */ accum = 1; wptr = pHTA->buf + strlen(pHTA->buf); if ((int) (pHTA->buf+pHTA->cbBuf - wptr) - 128 < (int) (2*(strlen(pHTA->pszAuthPass)+strlen(pHTA->pszAuthUser)))) { return "E-2-Request too large for buffer"; } ptr = pHTA->pszAuthUser; while(*ptr) { wptr += ToBase64(wptr,&accum,*ptr++); } wptr += ToBase64(wptr,&accum,':'); ptr = pHTA->pszAuthPass; while(*ptr) { wptr += ToBase64(wptr,&accum,*ptr++); } if (accum & 0x10000) { ToBase64(wptr,&accum,0); wptr[3] = '='; wptr += 4; } else if (accum & 0x100) { ToBase64(wptr,&accum,0); ToBase64(wptr,&accum,0); wptr[2] = '='; wptr[3] = '='; wptr += 4; } *wptr = '\0'; strcat(pHTA->buf,"\r\n"); } return 0; } EXPLAIN *HTTPaccess_SendRequest(struct HTTPaccess_s *pHTA, const char *pszRequest) { /* Returns error explanation */ /* Ensure connection */ int ret; char *reterr; if ((reterr=HTTPaccess_OpenConn(pHTA))) { if (bVERBOSE) { if (!bAUTOMATED) { printf("[s] 132 {%s}\n",reterr); } else { printf("%s\n",reterr); } } return "E-1-HTTPaccess_SendRequest: Could not open connection"; } /* Send request */ if (pHTA->ssl) { ret = SSL_write(pHTA->ssl,pszRequest,strlen(pszRequest)); } else { ret = writesocket(pHTA->s,pszRequest,strlen(pszRequest)); } if (ret == SOCKET_ERROR) { if (pHTA->bHTTP1_1) { return "E-3-HTTPaccess_SendRequest. HTTP/1.1 write error"; } return "E-2-HTTPaccess_SendRequest: Could not write socket"; closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } } return 0; } /* HTTPaccess_SendRequest */ EXPLAIN * HTTPaccess_ReadHead(struct HTTPaccess_s *pHTA,int *pcbHead) {/* Read from a socket, appending to a buffer until entire HTTP * header is obtained. (In HTTP the end of header is marked by * a blank line.) * Returns "E-3-..." if some problem which indicates a retry is * indicated. * Returns "E-1-..." if some problem which indicates to not * retry. */ int cnt; char *ptr; while(1) { /* Until there is a blank line in the buffer, which * marks the end of headers */ /* See if a blank line in what was read so far */ ptr = pHTA->buf; if ((pHTA->cbInbuf > 0) && ((*ptr == '\r')||(*ptr == '\n')) ) { *pcbHead = 0; return 0; } while(ptr) { ptr = memchr(ptr,'\n',pHTA->buf+pHTA->cbInbuf-ptr); if (ptr) { if (ptr+1 < pHTA->buf + pHTA->cbInbuf) { ptr++; if ((*ptr == '\r')||(*ptr == '\n')) { *pcbHead = ptr - pHTA->buf; return 0; } } else { /* 8/27/99 critical bug fix */ break; /* Need to read more */ } } } if (pHTA->cbBuf - pHTA->cbInbuf <= 0) { /* Buffer won't hold all headers */ return "E-1-HTTPaccess_ReadHead. Excessive response header\n"; } if (pHTA->ssl) { cnt = SSL_read(pHTA->ssl, pHTA->buf+pHTA->cbInbuf, pHTA->cbBuf- pHTA->cbInbuf); } else { cnt = readsocket(pHTA->s, pHTA->buf+pHTA->cbInbuf, pHTA->cbBuf- pHTA->cbInbuf); } if (bVERBOSE) { printf("R%d",cnt); } if (cnt <= 0) { return "E-3-HTTPaccess_ReadHead. read error\n"; } pHTA->cbInbuf += cnt; if (strncasecmp(pHTA->buf,"HTTP/1.",7)) { /* No headers (or response not understood */ return "E-3-HTTPaccess_ReadHead. Confused headers\n"; } } } /* HTTPaccess_ReadHead */ EXPLAIN * HTTPaccess_ProcessHead(struct HTTPaccess_s *pHTA,int cbHead) { char *ptr; /*******************************************/ /* Process header part of response. */ pHTA->cbContent = HTTPaccess_UNTILCLOSE; /* Default: Until socket closes */ ptr = pHTA->buf; ptr += 7; if (pHTA->bHTTP1_1 && (*ptr != '0')) { /* HTTP 1.1 connections can be left open */ pHTA->bPERSIST = 1; } while(ptr < pHTA->buf + cbHead) { ptr = memchr(ptr,'\n',pHTA->buf + cbHead - ptr); if (!ptr) { return 0; } ptr++; if (!strncasecmp(ptr,"Transfer-Encoding: ",19)) { /* For HTTP/1.1 only */ ptr += 19; ptr += countbl(ptr); if (strncasecmp(ptr,"chunked",7)) { /* Don't know how to deal with anything else */ pHTA->bHTTP1_1 = 0; closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } pHTA->bPERSIST = 0; return "E-3-HTTPaccess_ProcessHead. Unsupported Transfer-Encoding\n"; } pHTA->cbContent = HTTPaccess_CHUNKED; } else if (!strncasecmp(ptr,"Connection: ",12)) { ptr += 12; ptr += countbl(ptr); if (!strncasecmp(ptr,"close",5)) { pHTA->bPERSIST = 0; if (bVERBOSE) { printf("[C]"); } } } else if (!strncasecmp(ptr,"Content-Length: ",16)) { ptr += 16; ptr += countbl(ptr); pHTA->cbContent = atoi(ptr); if (bVERBOSE) { printf("L%ld",pHTA->cbContent); } } if (*ptr == '\r') { ptr++; } if (*ptr == '\n') { ptr++; return 0; } } return 0; } /* HTTPaccess_ProcessHead */ EXPLAIN *HTTPaccess_ReadBody(struct HTTPaccess_s *pHTA, int (*fn)(void *,const char *,int), void *pID) /* passed to fn */ { /* Read a HTTP body from a buffer and socket. Unprocessed data left in the buffer is processed first, then data is read from the socket. Will stop reading when reach specified length (cbContent) or got a chunk-size of 0. */ char *ptr; int cbChunk; int cnt; if (pHTA->cbContent == 0) { return 0; } if (pHTA->cbContent == HTTPaccess_CHUNKED) { /* Chunked Transfer Coding. */ /* See RFC 2068 Section 3.6 and 19.4.6 */ cbChunk = 0; /* At start of chunk */ while(1) { if (cbChunk == 0) { /* Need to read chunk size */ ptr = memchr(pHTA->buf,'\n',pHTA->cbInbuf); if (!ptr) { goto readmore; } ptr++; cbChunk = strtol(pHTA->buf,0,16); if (bVERBOSE) { printf("C%d",cbChunk); } cnt = pHTA->cbInbuf - (ptr - pHTA->buf); /* Remainder */ if (cbChunk == 0) { /* End of chunks, start of entity headers */ if (cnt > 0) { /* Move rest of data down to base of buffer */ memmove(pHTA->buf,ptr,cnt); pHTA->cbInbuf = cnt; } if (HTTPaccess_ReadHead(pHTA,&cnt)) { return "E-3-HTTPaccess_ReadBody bad chunked encoding reading entity-headers"; } /* Discard headers */ ptr = pHTA->buf + cnt; if (*ptr == '\r') { ptr++; } ptr++; cnt = pHTA->cbInbuf + pHTA->buf - ptr; if (cnt > 0) { memmove(pHTA->buf,ptr,cnt); /* Move data down to base */ } pHTA->cbInbuf = cnt; return 0; } } else { /* Still processing a chunk */ ptr = pHTA->buf; cnt = pHTA->cbInbuf; } /* Here with ptr pointing to cnt bytes of unprocessed data */ if (cnt >= cbChunk + 2) { /* More data than chunk. There must be a CRLF following before processing can be done. */ if ((*fn)(pID,ptr,cbChunk)) { return "E-2-HTTPaccess_ReadBody callback error"; } ptr += cbChunk; cnt = pHTA->cbInbuf - (ptr - pHTA->buf); /* Remainder */ ptr = memchr(ptr,'\n',cnt); if (!ptr) { return "E-3-HTTPaccess_ReadBody bad chunked coding no CRLF at end of chunk"; } ptr++; cnt = pHTA->cbInbuf - (ptr - pHTA->buf); /* Remainder */ pHTA->cbInbuf = cnt; memmove(pHTA->buf,ptr,cnt); /* Move data down to base */ cbChunk = 0; continue; /* don't readmore yet.*/ } else { /* Write everything we have, possibly reserving 1 byte */ if (cbChunk - cnt < 1) { cnt--; /* preserve one byte */ } if ((*fn)(pID,ptr,cnt)) { return "E-2-HTTPaccess_ReadBody callback error"; } cbChunk -= cnt; pHTA->cbInbuf = ptr + cnt - (pHTA->buf + pHTA->cbInbuf); if (pHTA->cbInbuf > 0) { /* Move rest of data down to base of buffer */ memmove(pHTA->buf,ptr+cnt,pHTA->cbInbuf); } } readmore:; /* Need to read more */ if (pHTA->cbInbuf >= pHTA->cbBuf) { /* No room */ return "E-3-HTTPaccess_ReadBody bad chunked coding"; } if (pHTA->ssl) { cnt = SSL_read(pHTA->ssl, pHTA->buf+pHTA->cbInbuf, pHTA->cbBuf - pHTA->cbInbuf); } else { cnt = readsocket(pHTA->s, pHTA->buf+pHTA->cbInbuf, pHTA->cbBuf - pHTA->cbInbuf); } if (cnt < 0) { return "E-1-HTTPaccess_ReadBody socket read error"; } pHTA->cbInbuf += cnt; } } else if (pHTA->cbContent == HTTPaccess_UNTILCLOSE) { /* Read until close */ while(1) { if (pHTA->cbInbuf > 0) { /* Write what we have */ if ((*fn)(pID,pHTA->buf,pHTA->cbInbuf)) { return "E-2-HTTPaccess_ReadBody callback error"; } } if (pHTA->ssl) { cnt = SSL_read(pHTA->ssl,pHTA->buf,pHTA->cbBuf); } else { cnt = readsocket(pHTA->s,pHTA->buf,pHTA->cbBuf); } if (cnt < 0) { return "E-1-HTTPaccess_ReadBody socket read error"; } if (cnt == 0) { return 0; } pHTA->cbInbuf = cnt; } } else { /* Read cbContent */ while(1) { if (pHTA->cbInbuf >= pHTA->cbContent) { /* Write partial */ if ((*fn)(pID,pHTA->buf,pHTA->cbContent)) { return "E-2-HTTPaccess_ReadBody callback error"; } ptr = pHTA->buf + pHTA->cbContent; cnt = pHTA->cbInbuf - (ptr - pHTA->buf); /* Remainder */ if (cnt > 0) { memmove(pHTA->buf,ptr,cnt); } pHTA->cbInbuf = cnt; return 0; } else { /* write everything in the buffer */ if ((pHTA->cbInbuf > 0) && (*fn)(pID,pHTA->buf,pHTA->cbInbuf)) { /* 5-25-99 */ return "E-2-HTTPaccess_ReadBody callback error"; } pHTA->cbContent -= pHTA->cbInbuf; } cnt = pHTA->cbBuf; if (cnt > pHTA->cbContent) { cnt = pHTA->cbContent; } if (pHTA->ssl) { cnt = SSL_read(pHTA->ssl,pHTA->buf,cnt); } else { cnt = readsocket(pHTA->s,pHTA->buf,cnt); } if (cnt < 0) { return "E-1-HTTPaccess_ReadBody socket read error"; } pHTA->cbInbuf = cnt; } } } /* HTTPaccess_ReadBody */ EXPLAIN *HTTPaccess_Retrieve(struct HTTPaccess_s *pHTA, char *pszURI, int (*fn)(void *,const char *,int), void *pID) { /* This is a general purpose function itself, and serves as an example of how to create complex HTTP/1.1 requests The pID is used as the first argument to the fn, which should write the amount of data in the buffer, and return 0 for success, -1 for not. */ char *pszExplain; char *ptr; int errcode; int cnt; retryrequest: if (pHTA->cAttempt) { /* Retry handling */ /* A subsequent attempt due to HTTP read or write error which could have been a transient due to network problems, or it could be a poorly implemented proxy or cache. HTTP/1.1 is more efficient, but this will use HTTP/1.0 after a number of failures of HTTP/1.1 */ if (pHTA->s && pHTA->bPERSIST) { closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } } pHTA->bPERSIST = 0; if (pHTA->cAttempt <= 2) { if (bVERBOSE) { printf("Retry...\n"); } } else if (pHTA->bHTTP1_1) { pHTA->cAttempt = 0; pHTA->bHTTP1_1 = 0; if (bVERBOSE) { printf("Retry as HTTP/1.0...\n"); } } else { return "E-4-HTTPaccess_Retrieve failure after retries"; } } pHTA->cAttempt++; /* Format the request appropriately for the appropriate HTTP version and proxy (if any). */ if (HTTPaccess_GenRequest(pHTA,pszURI)) { if (bAUTOMATED) { printf("[f] 100 {Could not format request into buffer}\n"); } else { fprintf(stderr,"Terminated. Could not format request into buffer\n"); } exit(-1); } /**** Add additional headers, depending on HTTP version */ if (pHTA->bHTTP1_1) { strcat(pHTA->buf,"Content-Length: 0\r\n"); } /**** Add header separator, and nothing else (empty body) */ strcat(pHTA->buf,"\r\n"); if (bVERBOSE) { /* Show entire request headers*/ fprintf(stderr,"%s",pHTA->buf); } /**** Send the request */ pszExplain = HTTPaccess_SendRequest(pHTA,pHTA->buf); if (pszExplain) { if ((atoi(pszExplain+2) == 3)) { goto retryrequest; } else { if (bAUTOMATED) { printf("[s] 133 {%s}\n",pszExplain); } else { fprintf(stderr,"%s\n",pszExplain); } return "E-5-HTTPaccess_Retrieve failure"; } } /**** Receive response */ pHTA->cbInbuf = 0; pszExplain = HTTPaccess_ReadHead(pHTA,&cnt); if (pszExplain && (atoi(pszExplain+2) == 3)) { /* Retry if we got E-3 */ goto retryrequest; } pszExplain = HTTPaccess_ProcessHead(pHTA,cnt); if (pszExplain && (atoi(pszExplain+2) == 3)) { /* Retry if we got E-3 */ goto retryrequest; } /* Skip forward to response code */ ptr = pHTA->buf + counttowh(pHTA->buf); ptr += countbl(ptr); errcode = atoi(ptr); /**** Handle response */ if (errcode != 200) { ptr = strchr(pHTA->buf,'\n'); if (!ptr) { ptr = pHTA->buf + strlen(pHTA->buf); } if (bAUTOMATED) { printf("[s] 101 {HTTP status not OK} %s %d\n", pszURI,errcode); } else { printf("Error: %s HTTP status %d", pszURI,errcode); fwrite(pHTA->buf,1,ptr-pHTA->buf,stdout); printf("\n"); } pHTA->bPERSIST = 0; closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } return "E-1-HTTPaccess_Retrieve Status code is not 200"; } /*******************************************/ /* Skip past headers and read the rest of the body */ ptr = pHTA->buf + cnt; if (*ptr == '\r') { ptr++; } ptr++; cnt = pHTA->buf + pHTA->cbInbuf - ptr; if (cnt > 0) { memmove(pHTA->buf,ptr,cnt); } pHTA->cbInbuf = cnt; if (HTTPaccess_ReadBody(pHTA,fn,pID)) { /* If we get an error here, we have to let the caller retry, since some information may have been written using the fn and pID. */ closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } pHTA->s = 0; pHTA->bPERSIST = 0; return "E-3-HTTPaccess_Retrieve error during ReadBody"; } /*******************************************/ /* Finish up this transfer. */ if (!pHTA->bPERSIST) { closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } pHTA->s = 0; } return 0; } /* HTTPaccess_Retrieve */ /* End of HTTPaccess_ routines */ /**************************************************************/ /**************************************************************/ /* The packing list is implemented as a contiguous block of memory which is realloc'ed to add entries. Append onto the block with PackListAppend(), this does not need to be on line boundaries. PackListNext() is used to enumerate the non-comment entries in the loaded packing list. */ char *pPacklist = 0; size_t cbPacklist = 0; int PackListAppend(void *pUnused,const char *buf,int amt) { if (amt <= 0) { return -1; } pPacklist = realloc(pPacklist,cbPacklist + amt + 1); if (!pPacklist) { if (bAUTOMATED) { printf("[f] 102 {Memory allocation failure} %ld\n", (long) cbPacklist + amt + 1); } else { fprintf(stderr,"Terminated: Could not realloc %ld\n", (long) cbPacklist + amt + 1); } exit(-1); } memcpy(pPacklist+cbPacklist,buf,amt); cbPacklist += amt; pPacklist[cbPacklist] = 0; /* Always ensure there is NUL terminator */ if (bAUTOMATED) { printf("[s] 110 {Packing List Progress Report} %ld\n",(long) cbPacklist); fflush(stdout); } return 0; } /* PackListAppend */ const char *PackListNext(const char *pLast,int bWantComments) { /* Iterator for accessing a packing list */ const char *pNext; if (!pLast) { pNext = pPacklist; } else { /* Advance first */ pNext = strchr(pLast,'\n'); if (pNext) { pNext++; } } while(1) { if (!pNext || (pNext >= pPacklist + cbPacklist)) { return 0; } if ((*pNext == '#')|| (*pNext == '\r')|| (*pNext == '\n')) { /* Ignore it as a comment */ if (!strncasecmp(pNext,"#-#httpsync ",12)) { USEVER = atoi(pNext + 12); if (USEVER > PACKLISTVER) { if (bAUTOMATED) { printf("[f] 103 {Packing list requires newer version of HTTPsync}"); } else { fprintf(stderr,"Terminated: This packing list requires newer version of HTTPsync.\n"); } exit(-1); } } if (bWantComments) { return pNext; } pNext = strchr(pNext,'\n'); if (!pNext) { return 0; } pNext++; /* loop back around */ } else if (memstr(pNext,"..",counttoch(pNext,'\n'))) { /* Disallow */ if (bAUTOMATED) { printf("[f] 104 {Packing list may not contain '..' in file names}\n"); } else { fprintf(stderr,"Terminated. Packing list may not contain \"..\" in file names\n"); } exit(-1); } else if ((*pNext == '.')&&(pNext[1] == '/')) { return pNext; /* A file specifier */ } else if ((*pNext == 'R')&&((pNext[1] == '/')||!strncmp(pNext+1,"http",4))) { return pNext; /* A redirected URL */ } else if ((*pNext == 'O')&&(pNext[1] == '.')) { return pNext; /* Obsolete file or directory */ } else { if (bAUTOMATED) { printf("[f] 105 {Packing list includes unrecognized line.}\n"); } else { fprintf(stderr,"Terminated: Packing list syntax. Unrecognized line.\n"); } exit(-1); } } } /* PackListNext */ char *PackListCheckRedirect(char *pszURI) { const char *ptr = pPacklist; const char *ptr2; char *aszRet = 0; while(ptr) { if (*ptr == 'R') { /* redirect URL. There is assumed only one 'R'. */ ptr2 = strchr(ptr,'\n'); if (!ptr2) { ptr2 = ptr + strlen(ptr); } astrn0cpy(&aszRet,ptr+1,ptr2-(ptr+1)); if (!aszRet) { if (bAUTOMATED) { printf("[f] 106 {Memory allocation failure}\n"); } else { fprintf(stderr,"Terminated: Could not malloc\n"); } exit(-1); } return aszRet; } ptr = strchr(ptr,'\n'); if (ptr) { ptr++; } } return pszURI; /* No change */ } /* PackListCheckRedirect */ /**************************************************************/ /**************************************************************/ int mainModReport(int argc,char **argv) { /* Generate a version 1.01 packing list from a list of files on stdin. (HTTPsync 1.12 uses version 1.01 packing lists.) For each listed file, write out file name, length, GMT time stamp, and modes. Files names must start with './' and not include '..' */ char buf[8192]; const char *pNext; char *pszDest = 0; int cnt; struct stat s; static char *szDays[]= { "Sun", "Mon" , "Tue" , "Wed" , "Thu" , "Fri" , "Sat" }; static char *szMonths[] = {"Jan" , "Feb" , "Mar" , "Apr" , "May" , "Jun" , "Jul" , "Aug" , "Sep" , "Oct" , "Nov" , "Dec"}; /* Read it in */ while((cnt = read(fileno(stdin),buf,sizeof(buf))) > 0) { PackListAppend(0,buf,cnt); } pNext = PackListNext(0,1); if (pNext && (*pNext != '#')) { /* If packing list doesn't start with a comment, then it is probably a plain list. We add HTTPsync a version specifier: 101 for the packing list we are generating. */ printf("#-#httpsync 101 Packing List for httpsync 1.01-1.14\n"); printf("# Visit the httpsync home page: http://www.mibsoftware.com/httpsync/\n"); } while (pNext) { /* PackListNext() is validating lines and will terminate if there is any trouble with file names, format, unrecognized line types, etc. The routine here modifies only the lines beginning with '.', the rest are written unchanged. */ if (*pNext == '.') { /* Find end of file name */ /* Make a copy of just the file name */ astrn0cpy(&pszDest,pNext,counttowh(pNext)); if (!pszDest) { if (bAUTOMATED) { printf("[f] 128 {Memory allocation failure}\n"); } else { fprintf(stderr,"Terminated: Could not realloc %d\n", counttowh(pNext)); } exit(-1); } if (USEVER >= 200) { URIunescape(pszDest); } if (!stat(pszDest,&s)) { if (s.st_mode & S_IFDIR) { } else { struct tm *tmptr; tmptr = gmtime(&s.st_mtime); fwrite(pNext,1,counttowh(pNext),stdout); printf(" %ld %s, %02d %s %04d %02d:%02d:%02d GMT %o\n", (long) s.st_size, /* Cast from off_t Thx to Andrew Vasilyev */ szDays[tmptr->tm_wday], tmptr->tm_mday, szMonths[tmptr->tm_mon], tmptr->tm_year+1900, tmptr->tm_hour, tmptr->tm_min, tmptr->tm_sec, (int) (s.st_mode & GENMODMASK)); } } else { /* can't stat. must be obsolete. */ printf("O%s\n", pszDest); } } else { /* write it unchanged */ fwrite(pNext,1,counttoch(pNext,'\n'),stdout); fwrite("\n",1,1,stdout); } pNext = PackListNext(pNext,1); } if (pszDest) { free(pszDest); } return 0; } /* mainModReport */ /**************************************************************/ int mainHTTPSYNC(struct HTTPaccess_s *pHTA); int main(int argc,char **argv) { int argind = 1; struct HTTPaccess_s theHTA; struct HTTPaccess_s *pHTA = &theHTA; char buf[8192]; if (argc < 2) { if (bAUTOMATED) { printf("[f] 107 {Usage error. httpsync V 3.00. Copyright 2001 Mib Software www.mibsoftware.com.}\n"); } else { fprintf(stderr,"httpsync V 3.00. Copyright 2001 Mib Software www.mibsoftware.com.\n"); fprintf(stderr,"Usage (get update): httpsync [-x proxy-host] [-u user -p pass] [-a] [-t tempdir] @http://host/URI\n"); fprintf(stderr,"Usage (create list): httpsync -m packing.lst\n"); } exit(-1); } if (!strcmp(argv[1],"-m")) { return mainModReport(argc,argv); } HTTPaccess_Initialize(pHTA,buf,sizeof(buf)); astrn0cpy(&aszWorkDest,"",0); cbTempdir = 0; while((argind < argc) && (argv[argind][0] == '-')) { switch(argv[argind][1]) { case 'u': /* 10/11/2001 */ if (!argv[argind][2]) { argind++; pHTA->pszAuthUser = argv[argind]; } else { pHTA->pszAuthUser = argv[argind]+2; } argind++; break; case 'p': /* 10/11/2001 */ if (!argv[argind][2]) { argind++; pHTA->pszAuthPass = strdup(argv[argind]); } else { pHTA->pszAuthPass = strdup(argv[argind]+2); } /* FIXME: nothing frees pHTA->pszAuthPass */ memset(argv[argind],0,strlen(argv[argind])); /* Might hide password on some systems */ argv[argind] = 0; /* Might hide password on some systems */ argind++; break; case 'x':/* Use a proxy */ if (!argv[argind][2]) { argind++; pHTA->pszProxy = argv[argind]; } else { pHTA->pszProxy = argv[argind]+2; } argind++; break; case 'a': /* 3/13/2000 */ /* Automated interface. Easy to parse status output */ bAUTOMATED = 1; argind++; break; case 't': /* 3/13/2000 */ if (!argv[argind][2]) { argind++; astrn0cpy(&aszWorkDest,argv[argind],strlen(argv[argind])); } else { astrn0cpy(&aszWorkDest,argv[argind]+2,strlen(argv[argind]+2)); } if (aszWorkDest[strlen(aszWorkDest)-1] != '/') { astrn0cat(&aszWorkDest,"/",1); } cbTempdir = strlen(aszWorkDest); argind++; break; case 'v': /* Mostly useful for debugging. Not documented. */ bVERBOSE = 1; argind += 1; break; default: if (bAUTOMATED) { printf("[f] 108 {Unknown option} '%c'\n",argv[argind][1]); } else { fprintf(stderr,"Terminated: unknown option '%c'\n",argv[argind][1]); } exit(-1); break; } } if (pHTA->pszAuthUser && !pHTA->pszAuthPass) { printf("Password: ");fflush(stdout); #if defined(WIN32) { int i = 0; int ch; do { ch = _getch(); if ((ch == 0x7f)||(ch == 0x08)) { if (i > 0) { i--; } } else if (ch && (i < sizeof(buf)-2)) { buf[i++] = ch; } } while(ch != '\r'); pHTA->pszAuthPass = strdup(buf); } #else pHTA->pszAuthPass = strdup(getpass("")); #endif } /* Get current umask */ umaskval = umask(0); umask(umaskval); if (HTTPaccess_ParseURL(pHTA,argv[argind]+1,pHTA->pszProxy)) { if (bAUTOMATED) { printf("[f] 109 {Invalid URL syntax}\n"); } else { fprintf(stderr,"Terminated: URL syntax\n"); } exit(-1); } /* End of command line parameter processing */ /********************************************/ return mainHTTPSYNC(pHTA); } /* main */ /**************************************************************/ /**************************************************************/ int fnWriteFile(void *v,const char *p,int cb) { /* Helper function */ FILE *f = (FILE *) v; if (write(fileno(f),p,cb) >= 0) { if (bAUTOMATED) { cbCompleted += cb; printf("[s] 118 {Progress Report} %ld %ld %ld %ld\n", cbCompleted,cbAllFiles,cFileCompleted,cAllFiles); fflush(stdout); } return 0; } return -1; } /**************************************************************/ const char *TransferDecide(const char *pszDest, const char *ptr, time_t *ptLastmod, int *pbvMode); /* helper */ void EnsureDirectoryForFile(const char *pszDest); /* helper */ int mainHTTPSYNC(struct HTTPaccess_s *pHTA) { /* Main loop */ time_t tLastmod; long cbLastmod = 0; int cUnmodified = 0; char *pszFinalDest; const char *pNext; FILE *fDEST; const char *ptr; const char *pszTransferReason =""; int bvMode; struct stat s; int cURIprefix; /* Accumulators for bAUTOMATED */ int bScanAction = 0; SocketStartup(); #ifdef SSLeay_add_ssl_algorithms SSL_library_init(); #endif SSL_load_error_strings(); pszFinalDest = ""; /* Retrieve and store the packing list in memory. The packing list may specify a different file name than the URL, so the packing list is written to a file after retrieval is complete. */ if (bAUTOMATED) { printf("[s] 129 {Starting Packing List} {%s}\n", pszURI); fflush(stdout); } else { printf("GET %s...", pszURI); } pHTA->cAttempt = 0; while(1) { fflush(stdout); /* Initialize for read */ cbPacklist = 0; ptr = HTTPaccess_Retrieve(pHTA,pszURI,PackListAppend,0); if (!ptr) break; /* success */ if (atoi(ptr+2)!=3) { if (bAUTOMATED) { printf("[f] 130 {Can't get packing list.} {%s} {%s}\n",pszURI,ptr); } else { fprintf(stderr,"Terminated. can't get packing list\n"); } exit(-1); } /* retry */ } if (bAUTOMATED) { printf("[s] 127 {Packing List Complete}\n"); fflush(stdout); } else { printf("OK\n"); } fflush(stdout); ptr = pszURI; pszURI = PackListCheckRedirect(pszURI); if ((pszURI != ptr) && !strncmp(pszURI,"http",4)) { /* Shutdown the connection and restart */ closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); pHTA->ssl =0 ; } pHTA->s = 0; pHTA->bPERSIST = 0; pHTA->bSKIPDNS = 0; if (HTTPaccess_ParseURL(pHTA,pszURI,pHTA->pszProxy)) { if (bAUTOMATED) { printf("[f] 134 {Redirect URL syntax}\n"); fflush(stdout); } else { fprintf(stderr,"Terminated. Redirect URL syntax\n"); } exit(-1); } } /* Copy the URI into an allocated buffer to allow strcat of the filenames later. */ ptr = pszURI; pszURI = 0; astrn0cpy(&pszURI,ptr,strlen(ptr)); /* Save the file */ ptr = strrchr(pszURI,'/'); if (!ptr) { if (bAUTOMATED) { printf("[f] 111 {No '/' in packing list URI}\n"); } else { fprintf(stderr,"Terminated: No '/' in packing list URI\n"); } exit(-1); } ptr++; aszWorkDest[cbTempdir] = '\0'; astrn0cat(&aszWorkDest,ptr,strlen(ptr)); if (cbTempdir) { EnsureDirectoryForFile(aszWorkDest); } fDEST = fopen(aszWorkDest,FOPEN_WRITE_BINARY); if (!fDEST) { if (bAUTOMATED) { printf("[f] 112 {Can't open for writing} {%s}\n",aszWorkDest); } else { fprintf(stderr,"Terminated: can't save '%s'. (Packing list missing 'R'?)\n",aszWorkDest); } exit(-1); } if (fwrite(pPacklist,1,cbPacklist,fDEST) != cbPacklist) { if (bAUTOMATED) { printf("[f] 113 {Write error} {%s}\n",ptr); } else { fprintf(stderr,"Terminated: can't save '%s'. write error\n",ptr); } exit(-1); } fclose(fDEST); if (bAUTOMATED) { if (cbTempdir) { printf("[a] 120 {mv required} {%s} {%s}\n",aszWorkDest,aszWorkDest+cbTempdir); fflush(stdout); } } cURIprefix = ptr - pszURI; /* pszURI is a prefix. Packing list filenames will be appended at this point */ if (bAUTOMATED) { bScanAction = 1; } pNext = PackListNext(0,0); while(pNext) { /* For each entry in the packing list */ fDEST = 0; tLastmod = 0; cbLastmod = -1; bvMode = 0; pszURI[cURIprefix] = '\0'; /* 5-25-99 Modified to eliminate /./ requests which * are rejected by Netscape enterprise server */ astrn0cat(&pszURI,pNext+2,counttowh(pNext+2)); aszWorkDest[cbTempdir] = '\0'; astrn0cat(&aszWorkDest,pszURI + cURIprefix,strlen(pszURI + cURIprefix)); pszFinalDest = aszWorkDest + cbTempdir; if (USEVER >= 200) { URIunescape(pszFinalDest); } ptr = pNext + counttowh(pNext); ptr += countbl(ptr); if (*pNext == 'R') { /* Loop back around 'R' was already processed */ } else if (*pNext == 'O') { /* Remove obsolete file */ if (!bScanAction) { if (!stat(pszFinalDest+1,&s)) { /* It exists */ if (s.st_mode & S_IFDIR) { if (bAUTOMATED) { printf("[a] 115 {rmdir required} {%s}\n",pszFinalDest+1); fflush(stdout); } else { printf("Removing obsolete directory: %s\n", pszFinalDest+1); if (rmdir(pszFinalDest+1)) { printf("...Warning: Directory not removed: %s\n", strerror(errno)); } } } else { if (bAUTOMATED) { printf("[a] 116 {rm required} {%s}\n",pszFinalDest+1); fflush(stdout); } else { printf("Removing obsolete file: %s\n",pszFinalDest+1); if (unlink(pszFinalDest+1)) { printf("...Warning: File not removed: %s\n",strerror(errno)); } } } } } } else if (*pNext == '.') { bvMode = 0777 & ~umaskval; /* default */ pszTransferReason = TransferDecide(pszFinalDest,ptr,&tLastmod,&bvMode); if (!pszTransferReason) { /* Don't transfer. It matches */ if (!bScanAction) { cUnmodified++; } } else if (!strcmp(pszTransferReason,"M")) { /* Just a mode change */ if (!bScanAction) { if (bAUTOMATED) { printf("[a] 121 {chmod required} 0%o {%s}\n",bvMode & ~umaskval,pszFinalDest); fflush(stdout); } else { printf("%s %s\n",pszTransferReason,pszFinalDest); chmod(aszWorkDest,bvMode & ~umaskval); /* Attempt it */ } } } else { if (bScanAction) { cAllFiles++; cbAllFiles += atol(ptr); } else { long markcbCompleted = cbCompleted; /* Otherwise: transfer is required */ if (bAUTOMATED) { printf("[s] 122 {Starting} {%s}\n",pszFinalDest); fflush(stdout); } else { printf("%s %s...", pszTransferReason,pszFinalDest); } fflush(stdout); pHTA->cAttempt = 0; while(1) { cbCompleted = markcbCompleted; fDEST = fopen(aszWorkDest,FOPEN_WRITE_BINARY); if (!fDEST) { /* Fix common reasons file could not be opened */ if ((bvMode & 0600) == 0400) { /* Exists as 04xx read only */ chmod(aszWorkDest,bvMode | 0200); } /* Ensure the directory */ EnsureDirectoryForFile(aszWorkDest); fDEST = fopen(aszWorkDest,FOPEN_WRITE_BINARY); } if (!fDEST) { if (bAUTOMATED) { printf("[f] 117 {Could not open for writing} {%s}\n", aszWorkDest); } else { fprintf(stderr, "Terminated. Could not open %s for writing\n", aszWorkDest); } exit(-1); } ptr = HTTPaccess_Retrieve(pHTA,pszURI,fnWriteFile,fDEST); fclose(fDEST); if (!ptr) break; /* success */ if (atoi(ptr+2)!=3) { if (bAUTOMATED) { printf("[f] 131 {Failed to transfer} {%s}\n",pszFinalDest); } else { fprintf(stderr,"Terminated. Could not transfer %s\n",pszFinalDest); } exit(-1); } /* retry */ if (bAUTOMATED) { printf("[s] 119 {Retrying} {%s}\n",pszFinalDest); fflush(stdout); } } if (bAUTOMATED) { cFileCompleted++; printf("[s] 118 {Progress Report} %ld %ld %ld %ld\n", cbCompleted,cbAllFiles,cFileCompleted,cAllFiles); fflush(stdout); } /* Fix timestamp and mode bits */ if (tLastmod) { struct utimbuf u; u.actime = time(NULL); u.modtime = tLastmod; utime(aszWorkDest,&u); } chmod(aszWorkDest,bvMode & ~umaskval); if (bAUTOMATED) { if (cbTempdir) { printf("[a] 120 {mv required} {%s} {%s}\n",aszWorkDest,pszFinalDest); fflush(stdout); } } else { printf("OK\n"); } } } } /* Advance */ pNext = PackListNext(pNext,0); if (!pNext && bAUTOMATED && bScanAction) { /* Loop back around */ pNext = PackListNext(0,0); bScanAction = 0; } } if (bAUTOMATED) { printf("[s] 123 {Skipped unchanged files} %d\n",cUnmodified); fflush(stdout); } else { printf("Skipped %d unchanged files\n",cUnmodified); } if (pHTA->s && pHTA->bPERSIST) { closesocket(pHTA->s); if (pHTA->ssl) { SSL_free (pHTA->ssl); SSL_CTX_free (pHTA->ctx); } } SocketCleanup(); return 0; } /* mainHTTPsync */ /**************************************************************/ /**************************************************************/ /******** Supporting Functions ********************************/ /**************************************************************/ const char *TransferDecide(const char *pszDest, const char *ptr, time_t *ptLastmod, int *pbvMode) { /* Make the decision to transfer or not transfer a file. pszDest is a file name. ptr is the pointer to the packing list information: length, time stamp, etc return 0 if unmodified, otherwise a short string which notes one reason the local doesn't match the remote. */ int cbLastmod = atoi(ptr); /* Length */ struct stat s; ptr = strchr(ptr,' '); /* Skip ahead */ if (!ptr) { /* Packing list syntax */ if (bAUTOMATED) { printf("[f] 124 {Packing file status information missing}\n"); } else { fprintf(stderr,"Terminated: Packing list syntax. File status information missing\n"); } exit(-1); } ptr += countbl(ptr); if (strchr(ptr,',')) { ptr = strchr(ptr,',')+1; ptr += countbl(ptr); } ptr += ssgmtime(ptr,ptLastmod); ptr += countbl(ptr); if (*ptr <= ' ') { if (bAUTOMATED) { printf("[f] 125 {Packing file status information missing}\n"); } else { fprintf(stderr,"Terminated: Packing list syntax. File status information missing\n"); } exit(-1); } /* Permissions */ *pbvMode = strtol(ptr,(char **) &ptr,8); *pbvMode &= 0777; if (!stat(pszDest,&s)) { if (s.st_mtime < *ptLastmod) { return("D.lt"); } else if(s.st_mtime - *ptLastmod >= TDIFFLIMIT) { return("D.gt"); } else if (s.st_size != cbLastmod) { return("S.ne"); } else { /* Everything else matches. Check perms */ if (*pbvMode && ((s.st_mode & 0700) != (*pbvMode & MODMASK))) { return("M"); } if (bVERBOSE) { printf("%s Unmodified from packing list\n", pszDest); } return 0; } } else { return("N"); } } /* TransferDecide */ /**************************************************************/ /**************************************************************/ void EnsureDirectoryForFile(const char *pszDest) { /* Create each directory in a chain. Ignore errors, since it is likely that we will run mkdir() on directories which already exist. */ char *work = 0; char *ptr; astrn0cpy(&work,pszDest,strlen(pszDest)); if (!work) { if (bAUTOMATED) { printf("[f] 126 {Memory allocation failure}\n"); } else { fprintf(stderr,"Terminated. Can't realloc\n"); } exit(-1); } ptr = work; while((ptr = strchr(ptr,'/'))) { *ptr = 0; portable_mkdir(work,0755 & ~umaskval); *ptr++ = '/'; } free(work); } /* EnsureDirectoryForFile */ /**************************************************************/ /** ssgmtime() For HTTPsync, it is known that the date string form will always be DD 3-letter-Month YYYY HH:MM:SS GMT The day of week was skipped before calling here. This loads the values into a struct tm, and then converts to a time_t. mktime() is not used since we can't figure out timezone in a portable way that is reliable across daylight savings, etc. **/ int ssgmtime(const char *sptr,time_t *time) { const char *ptr; struct tm tmbuf; static char *szMonths[] = {"Jan" , "Feb" , "Mar" , "Apr" , "May" , "Jun" , "Jul" , "Aug" , "Sep" , "Oct" , "Nov" , "Dec"}; ptr = sptr; /* Parse for the date. */ tmbuf.tm_mday = strtol(ptr,(char **) &ptr,10); ptr += countbl(ptr); for(tmbuf.tm_mon = 0;tmbuf.tm_mon < 12;tmbuf.tm_mon++) { if (!strncasecmp(szMonths[tmbuf.tm_mon],ptr,3)) { break; } } if (tmbuf.tm_mon >= 12) { return 0; } ptr += 3; ptr += countbl(ptr); tmbuf.tm_year = strtol(ptr,(char **) &ptr,10) - 1900; tmbuf.tm_hour = strtol(ptr,(char **) &ptr,10); if (*ptr != ':') { return 0; } tmbuf.tm_min = strtol(ptr+1,(char **) &ptr,10); if (*ptr != ':') { return 0; } tmbuf.tm_sec = strtol(ptr+1,(char **) &ptr,10); ptr += countbl(ptr); if ((ptr[0]=='G')&&(ptr[1]=='M')&&(ptr[2]=='T')&&isspace(ptr[3])) { ptr += 3; } else { return 0; } *time = tm_to_time(&tmbuf); return ptr - sptr; } /* ssgmtime */ /*********************************************************/ /*********************************************************/ /* Automatic allocating string functions */ char *astrn0cpy(char **ppasz,const char *src,int len) { /* allocate and strncpy, always store '\0' */ *ppasz = realloc(*ppasz,len+1); if (!*ppasz) { return 0; } strncpy(*ppasz,src,len); (*ppasz)[len] = 0; return *ppasz; } /* astrn0cpy */ char *astrn0cat(char **ppasz,const char *src,int len) { /* reallocate and strncat, always store '\0' */ int cBefore = 0; if (*ppasz) { cBefore = strlen(*ppasz); } *ppasz = realloc(*ppasz,len+1+cBefore); if (!*ppasz) { return 0; } strncat(*ppasz+cBefore,src,len); (*ppasz)[cBefore + len] = 0; return *ppasz; } /* astrn0cat */ /*********************************************************/ /*********************************************************/ /* Simple consecutive character counting functions */ int counttowh(const char *pStart) { const unsigned char *ptr; ptr = (const unsigned char *) pStart; while(*ptr && (*ptr > ' ')) { ptr++; } return (char *) ptr - pStart; } /* counttowh */ int countbl(const char *pStart) { const char *ptr; ptr = pStart; while((*ptr == ' ')||(*ptr == '\t')) { ptr++; } return (int) (ptr - pStart); } /* countbl */ int counttoch(const char *pStart,char ch) { const char *ptr = pStart; while(*ptr && (*ptr != ch)) { ptr++; } return ptr - pStart; } /* counttoch */ /**************************************************************/ /**************************************************************/ char *URIunescape(char *pStart) { /* Collapse %xx sequences in place */ char *ptr = pStart; char *ptr2; ptr = strchr(ptr,'%'); if (!ptr) { return pStart; } ptr2 = ptr; while(*ptr) { if (*ptr == '%') { ptr++; if (*ptr > '9') { *ptr2 = (*ptr & 0x0f) + 9; } else { *ptr2 = (*ptr & 0x0f); } ptr++; (*ptr2) <<= 4; if (*ptr > '9') { *ptr2 += (*ptr & 0x0f) + 9; } else { *ptr2 += (*ptr & 0x0f); } } else { *ptr2 = *ptr; } ptr++; ptr2++; } *ptr2 = '\0'; return pStart; } /* URIunescape */ /**************************************************************/ /**************************************************************/ char *memstr(const char *pBlock,const char *pSubstring,int cbBlock) { /* Added for 1.14 */ const char *ptr = pBlock; const char *end = pBlock + cbBlock; int len = strlen(pSubstring); while((ptr = memchr(ptr,*pSubstring,end - ptr))) { if (end - ptr < (int) strlen(pSubstring)) { return 0; } if (!strncmp(ptr,pSubstring,len)) { return (char *) ptr; } ptr++; } return 0; } /* memstr */ /**************************************************************/ /**************************************************************/ /* This tm_to_time software obtained from comp.sources.unix archive http://sources.isc.org/dirlist.perl?dir=devel/func/time/&tarball=tm_to_time/tm_to_time/tm_to_time.c The timezone correction is not needed for HTTPsync, and is "#if 0" removed. The register keywords in function nday() original were dropped. */ /**************************************************************/ #include /* Return 1 if `y' is a leap year, 0 otherwise. */ static int leap (int y) { y += 1900; if (y % 400 == 0) return (1); if (y % 100 == 0) return (0); return (y % 4 == 0); } /* Return the number of days between Jan 1, 1970 and the given * broken-down time. */ static int ndays (struct tm *p) { int n = p->tm_mday; int m, y; char *md = "\37\34\37\36\37\36\37\37\36\37\36\37"; for (y = 70; y < p->tm_year; ++y) { n += 365; if (leap (y)) ++n; } for (m = 0; m < p->tm_mon; ++m) n += md[m] + (m == 1 && leap (y)); return (n); } /* Convert a broken-down time (such as returned by localtime()) * back into a `time_t'. */ time_t tm_to_time (struct tm *tp) { #if 0 /* Don't need it in httpsync */ register int m1, m2; #endif time_t t; #if 0 /* Don't need it in httpsync */ struct tm otm; #endif t = (ndays (tp) - 1) * 86400L + tp->tm_hour * 3600L + tp->tm_min * 60 + tp->tm_sec; #if 0 /* Don't need it in httpsync */ /* * Now the hard part -- correct for the time zone: */ otm = *tp; tp = localtime (&t); m1 = tp->tm_hour * 60 + tp->tm_min; m2 = otm.tm_hour * 60 + otm.tm_min; t -= ((m1 - m2 + 720 + 1440) % 1440 - 720) * 60L; #endif return (t); } /**************************************************************/ /* This strtol software obtained from FreeBSD cvsweb */ /**************************************************************/ /*- * Copyright (c) 1990, 1993 * The Regents of the University of California. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #if defined(LIBC_SCCS) && !defined(lint) static char sccsid[] = "@(#)strtol.c 8.1 (Berkeley) 6/4/93"; #endif /* LIBC_SCCS and not lint */ #include #include #include #include /* * Convert a string to a long integer. * * Ignores `locale' stuff. Assumes that the upper and lower case * alphabets and digits are each contiguous. */ long strtol(nptr, endptr, base) const char *nptr; char **endptr; register int base; { register const char *s = nptr; register unsigned long acc; register unsigned char c; register unsigned long cutoff; register int neg = 0, any, cutlim; /* * Skip white space and pick up leading +/- sign if any. * If base is 0, allow 0x for hex and 0 for octal, else * assume decimal; if base is already 16, allow 0x. */ do { c = *s++; } while (isspace(c)); if (c == '-') { neg = 1; c = *s++; } else if (c == '+') c = *s++; if ((base == 0 || base == 16) && c == '0' && (*s == 'x' || *s == 'X')) { c = s[1]; s += 2; base = 16; } if (base == 0) base = c == '0' ? 8 : 10; /* * Compute the cutoff value between legal numbers and illegal * numbers. That is the largest legal value, divided by the * base. An input number that is greater than this value, if * followed by a legal input character, is too big. One that * is equal to this value may be valid or not; the limit * between valid and invalid numbers is then based on the last * digit. For instance, if the range for longs is * [-2147483648..2147483647] and the input base is 10, * cutoff will be set to 214748364 and cutlim to either * 7 (neg==0) or 8 (neg==1), meaning that if we have accumulated * a value > 214748364, or equal but the next digit is > 7 (or 8), * the number is too big, and we will return a range error. * * Set any if any `digits' consumed; make it negative to indicate * overflow. */ cutoff = neg ? -(unsigned long)LONG_MIN : LONG_MAX; cutlim = cutoff % (unsigned long)base; cutoff /= (unsigned long)base; for (acc = 0, any = 0;; c = *s++) { if (!isascii(c)) break; if (isdigit(c)) c -= '0'; else if (isalpha(c)) c -= isupper(c) ? 'A' - 10 : 'a' - 10; else break; if (c >= base) break; if (any < 0 || acc > cutoff || (acc == cutoff && c > cutlim)) any = -1; else { any = 1; acc *= base; acc += c; } } if (any < 0) { acc = neg ? LONG_MIN : LONG_MAX; errno = ERANGE; } else if (neg) acc = -acc; if (endptr != 0) *endptr = (char *)(any ? s - 1 : nptr); return (acc); }