This patch adds the --iconv option to rsync, which allows you to convert filenames from one charset to another. The option takes normal iconv options, plus "." to ask for the current locale setting, and "-" to turn off iconv conversion. You can specify the local and the remote setting separated by a comma, or you can specify a single term for both. For instance: --iconv=. Turn on conversions based on each rsync's locale --iconv=iso-8859-1 Both sides have charset ISO-8859-1 (so no con- version is done), but invalid names are skipped. --iconv=.,utf8 Default locale setting on local, UTF-8 on remote. --iconv=- Turn off all conversions (if not the default) To use this patch, run these commands for a successful build: patch -p1 dirname) != NULL) { + icnt = strlen(ibuf); + ocnt--; /* pre-subtract the space for the '/' */ + if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) + goto convert_error; + *obuf++ = '/'; + } + + ibuf = file->basename; + icnt = strlen(ibuf); + if (iconv(ic_send, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) { + convert_error: + io_error |= IOERR_GENERAL; + rprintf(FINFO, + "[%s] cannot convert filename: %s (%s)\n", + who_am_i(), f_name(file, fname), strerror(errno)); + return; + } + *obuf = '\0'; + } else +#endif + f_name(file, fname); flags = file->flags & XMIT_TOP_DIR; @@ -495,6 +529,9 @@ static struct file_struct *receive_file_ char thisname[MAXPATHLEN]; unsigned int l1 = 0, l2 = 0; int alloc_len, basename_len, dirname_len, linkname_len, sum_len; +#ifdef ICONV_OPTION + int ndx_len; +#endif OFF_T file_length; char *basename, *dirname, *bp; struct file_struct *file; @@ -529,7 +566,33 @@ static struct file_struct *receive_file_ read_sbuf(f, &thisname[l1], l2); thisname[l1 + l2] = 0; - strlcpy(lastname, thisname, MAXPATHLEN); + /* Abuse dirname_len for a moment... */ + dirname_len = strlcpy(lastname, thisname, MAXPATHLEN); + +#ifdef ICONV_OPTION + if (ic_recv != (iconv_t)-1) { + char *obuf = thisname, *ibuf = lastname; + size_t ocnt = MAXPATHLEN, icnt = dirname_len; + + if (icnt >= MAXPATHLEN) { + errno = E2BIG; + goto convert_error; + } + + iconv(ic_recv, NULL,0, NULL,0); + if (iconv(ic_recv, &ibuf,&icnt, &obuf,&ocnt) == (size_t)-1) { + convert_error: + io_error |= IOERR_GENERAL; + rprintf(FINFO, + "[%s] cannot convert filename: %s (%s)\n", + who_am_i(), lastname, strerror(errno)); + obuf = thisname; + } + *obuf = '\0'; + ndx_len = 4; + } else + ndx_len = 0; +#endif clean_fname(thisname, 0); @@ -550,6 +613,12 @@ static struct file_struct *receive_file_ dirname_len = 0; } basename_len = strlen(basename) + 1; /* count the '\0' */ +#ifdef ICONV_OPTION + if (basename_len == 1) { + basename = NULL; + basename_len = 0; + } +#endif file_length = read_longint(f); if (!(flags & XMIT_SAME_TIME)) @@ -599,6 +668,9 @@ static struct file_struct *receive_file_ sum_len = always_checksum && S_ISREG(mode) ? MD4_SUM_LENGTH : 0; alloc_len = file_struct_len + dirname_len + basename_len +#ifdef ICONV_OPTION + + ndx_len +#endif + linkname_len + sum_len; bp = pool_alloc(flist->file_pool, alloc_len, "receive_file_entry"); @@ -612,6 +684,13 @@ static struct file_struct *receive_file_ file->uid = uid; file->gid = gid; +#ifdef ICONV_OPTION + if (ndx_len) { + SIVAL(bp, 0, flist->count); + bp += ndx_len; + } +#endif + if (dirname_len) { file->dirname = lastdir = bp; lastdir_len = dirname_len - 1; @@ -648,8 +727,10 @@ static struct file_struct *receive_file_ } file->basename = bp; - memcpy(bp, basename, basename_len); - bp += basename_len; + if (basename_len) { + memcpy(bp, basename, basename_len); + bp += basename_len; + } if ((preserve_devices && IS_DEVICE(mode)) || (preserve_specials && IS_SPECIAL(mode))) @@ -926,7 +1007,7 @@ struct file_struct *make_file(char *fnam STRUCT_STAT st2; int save_mode = file->mode; file->mode = S_IFDIR; /* Find a directory with our name. */ - if (flist_find(the_file_list, file) >= 0 + if (flist_find(sorted_file_list, file) >= 0 && do_stat(thisname, &st2) == 0 && S_ISDIR(st2.st_mode)) { file->modtime = st2.st_mtime; file->length = st2.st_size; @@ -1304,11 +1385,17 @@ struct file_list *send_file_list(int f, flist->hlink_pool = NULL; } - /* Sort the list without removing any duplicates. This allows the + /* When converting names, the sender does not sort the file list + * because the names will differ on the sending and receiving sides + * (both sides will use the unsorted index number for each item). + * Otherwise, sort without removing any duplicates. This allows the * receiving side to ask for any name they like, which gives us the * flexibility to change the way we unduplicate names in the future * without causing a compatibility problem with older versions. */ - clean_flist(flist, 0, 0); +#ifdef ICONV_OPTION + if (!need_unsorted_flist) +#endif + clean_flist(flist, 0, 0); send_uid_list(f); @@ -1333,6 +1420,9 @@ struct file_list *recv_file_list(int f) struct file_list *flist; unsigned short flags; int64 start_read; +#ifdef ICONV_OPTION + struct file_struct **unsorted_list = NULL; +#endif rprintf(FLOG, "receiving file list\n"); if (show_filelist_p()) @@ -1377,8 +1467,31 @@ struct file_list *recv_file_list(int f) if (show_filelist_p()) finish_filelist_progress(flist); +#ifdef ICONV_OPTION + if (need_unsorted_flist) { + /* Save off the unsorted array of pointers so that all our + * index numbers can be into the original, unsorted list. We + * still need to create a sorted array so that the generator + * can use it (1) for wading through the files in sorted order, + * and (2) for calling flist_find(). */ + if (!(unsorted_list = new_array(struct file_struct *, flist->count)) + || !(sorted_file_list = new(struct file_list))) + out_of_memory("recv_file_list"); + memcpy(unsorted_list, flist->files, + flist->count * sizeof (struct file_struct*)); + } +#endif + clean_flist(flist, relative_paths, 1); +#ifdef ICONV_OPTION + if (need_unsorted_flist) { + *sorted_file_list = *flist; + flist->files = unsorted_list; + } else +#endif + sorted_file_list = flist; + if (f >= 0) { recv_uid_list(f, flist); @@ -1423,6 +1536,8 @@ int flist_find(struct file_list *flist, int low = flist->low, high = flist->high; int diff, mid, mid_up; + assert(!am_sender); + while (low <= high) { mid = (low + high) / 2; if (flist->files[mid]->basename) --- old/generator.c +++ new/generator.c @@ -91,7 +91,12 @@ extern char *backup_dir; extern char *backup_suffix; extern int backup_suffix_len; extern struct file_list *the_file_list; +extern struct file_list *sorted_file_list; extern struct filter_list_struct server_filter_list; +#ifdef ICONV_OPTION +extern int need_unsorted_flist; +extern unsigned int file_struct_len; +#endif static int deletion_count = 0; /* used to implement --max-delete */ @@ -99,6 +104,8 @@ static int deletion_count = 0; /* used t #define DEL_FORCE_RECURSE (1<<1) /* recurse even w/o --force */ #define DEL_TERSE (1<<3) +#define UNSORTED_NDX(f) ((int)IVAL((char*)(f)+file_struct_len,0)) + static int is_backup_file(char *fn) { @@ -215,8 +222,8 @@ static int delete_item(char *fname, int * MAXPATHLEN buffer with the name of the directory in it (the functions we * call will append names onto the end, but the old dir value will be restored * on exit). */ -static void delete_in_dir(struct file_list *flist, char *fbuf, - struct file_struct *file, STRUCT_STAT *stp) +static void delete_in_dir(char *fbuf, struct file_struct *file, + STRUCT_STAT *stp) { static int min_depth = MAXPATHLEN, cur_depth = -1; static void *filt_array[MAXPATHLEN/2+1]; @@ -225,7 +232,7 @@ static void delete_in_dir(struct file_li char delbuf[MAXPATHLEN]; int dlen, i; - if (!flist) { + if (!fbuf) { while (cur_depth >= min_depth) pop_local_filters(filt_array[cur_depth--]); min_depth = MAXPATHLEN; @@ -274,7 +281,7 @@ static void delete_in_dir(struct file_li struct file_struct *fp = dirlist->files[i]; if (!fp->basename || fp->flags & FLAG_MOUNT_POINT) continue; - if (flist_find(flist, fp) < 0) { + if (flist_find(sorted_file_list, fp) < 0) { f_name(fp, delbuf); delete_item(delbuf, fp->mode, DEL_FORCE_RECURSE); } @@ -285,7 +292,7 @@ static void delete_in_dir(struct file_li /* This deletes any files on the receiving side that are not present on the * sending side. This is used by --delete-before and --delete-after. */ -static void do_delete_pass(struct file_list *flist) +static void do_delete_pass(void) { char fbuf[MAXPATHLEN]; STRUCT_STAT st; @@ -295,8 +302,8 @@ static void do_delete_pass(struct file_l if (dry_run > 1 || list_only) return; - for (j = 0; j < flist->count; j++) { - struct file_struct *file = flist->files[j]; + for (j = 0; j < sorted_file_list->count; j++) { + struct file_struct *file = sorted_file_list->files[j]; if (!(file->flags & FLAG_DEL_HERE)) continue; @@ -309,9 +316,9 @@ static void do_delete_pass(struct file_l || !S_ISDIR(st.st_mode)) continue; - delete_in_dir(flist, fbuf, file, &st); + delete_in_dir(fbuf, file, &st); } - delete_in_dir(NULL, NULL, NULL, NULL); + delete_in_dir(NULL, NULL, NULL); if (do_progress && !am_server) rprintf(FINFO, " \r"); @@ -945,7 +952,7 @@ static void recv_generator(char *fname, rprintf(code, "%s/\n", fname); if (delete_during && f_out != -1 && !phase && dry_run < 2 && (file->flags & FLAG_DEL_HERE)) - delete_in_dir(the_file_list, fname, file, &st); + delete_in_dir(fname, file, &st); return; } @@ -1233,7 +1240,7 @@ static void recv_generator(char *fname, if (!dry_run) { char numbuf[4]; SIVAL(numbuf, 0, ndx); - send_msg(MSG_SUCCESS, numbuf, 4); + send_msg(MSG_SUCCESS, numbuf, 4, 0); } return; } @@ -1352,7 +1359,7 @@ static void recv_generator(char *fname, void generate_files(int f_out, struct file_list *flist, char *local_name) { - int i; + int i, ndx; char fbuf[MAXPATHLEN]; int itemizing, maybe_ATTRS_REPORT; enum logcode code; @@ -1389,7 +1396,7 @@ void generate_files(int f_out, struct fi } if (delete_before && !local_name && flist->count > 0) - do_delete_pass(flist); + do_delete_pass(); do_progress = 0; if (append_mode || whole_file < 0) @@ -1408,16 +1415,22 @@ void generate_files(int f_out, struct fi ignore_timeout = 1; for (i = 0; i < flist->count; i++) { - struct file_struct *file = flist->files[i]; + struct file_struct *file = sorted_file_list->files[i]; if (!file->basename) continue; +#ifdef ICONV_OPTION + ndx = need_unsorted_flist ? UNSORTED_NDX(file) : i; +#else + ndx = i; +#endif + if (local_name) strlcpy(fbuf, local_name, sizeof fbuf); else f_name(file, fbuf); - recv_generator(fbuf, file, i, itemizing, maybe_ATTRS_REPORT, + recv_generator(fbuf, file, ndx, itemizing, maybe_ATTRS_REPORT, code, f_out); /* We need to ensure that any dirs we create have writeable @@ -1447,7 +1460,7 @@ void generate_files(int f_out, struct fi } recv_generator(NULL, NULL, 0, 0, 0, code, -1); if (delete_during) - delete_in_dir(NULL, NULL, NULL, NULL); + delete_in_dir(NULL, NULL, NULL); phase++; csum_length = SUM_LENGTH; @@ -1465,13 +1478,13 @@ void generate_files(int f_out, struct fi /* files can cycle through the system more than once * to catch initial checksum errors */ - while ((i = get_redo_num(itemizing, code)) != -1) { - struct file_struct *file = flist->files[i]; + while ((ndx = get_redo_num(itemizing, code)) != -1) { + struct file_struct *file = flist->files[ndx]; if (local_name) strlcpy(fbuf, local_name, sizeof fbuf); else f_name(file, fbuf); - recv_generator(fbuf, file, i, itemizing, maybe_ATTRS_REPORT, + recv_generator(fbuf, file, ndx, itemizing, maybe_ATTRS_REPORT, code, f_out); } @@ -1503,7 +1516,7 @@ void generate_files(int f_out, struct fi do_progress = save_do_progress; if (delete_after && !local_name && flist->count > 0) - do_delete_pass(flist); + do_delete_pass(); if ((need_retouch_dir_perms || need_retouch_dir_times) && dir_tweaking) { int j = 0; @@ -1511,7 +1524,7 @@ void generate_files(int f_out, struct fi * modified during the transfer and/or re-set any tweaked * modified-time values. */ for (i = 0; i < flist->count; i++) { - struct file_struct *file = flist->files[i]; + struct file_struct *file = sorted_file_list->files[i]; if (!file->basename || !S_ISDIR(file->mode)) continue; @@ -1527,7 +1540,12 @@ void generate_files(int f_out, struct fi i--; continue; } - recv_generator(f_name(file, NULL), file, i, itemizing, +#ifdef ICONV_OPTION + ndx = need_unsorted_flist ? UNSORTED_NDX(file) : i; +#else + ndx = i; +#endif + recv_generator(f_name(file, NULL), file, ndx, itemizing, maybe_ATTRS_REPORT, code, -1); if (allowed_lull && !(++j % lull_mod)) maybe_send_keepalive(); --- old/hlink.c +++ new/hlink.c @@ -237,7 +237,7 @@ int hard_link_check(struct file_struct * if (remove_source_files == 1 && do_xfers) { char numbuf[4]; SIVAL(numbuf, 0, ndx); - send_msg(MSG_SUCCESS, numbuf, 4); + send_msg(MSG_SUCCESS, numbuf, 4, 0); } file->F_HLINDEX = FINISHED_LINK; } else @@ -307,7 +307,7 @@ void hard_link_cluster(struct file_struc if (remove_source_files == 1 && do_xfers) { char numbuf[4]; SIVAL(numbuf, 0, ndx); - send_msg(MSG_SUCCESS, numbuf, 4); + send_msg(MSG_SUCCESS, numbuf, 4, 0); } file->F_HLINDEX = FINISHED_LINK; } while (!(file->flags & FLAG_HLINK_EOL)); --- old/io.c +++ new/io.c @@ -51,6 +51,9 @@ extern int preserve_hard_links; extern char *filesfrom_host; extern struct stats stats; extern struct file_list *the_file_list; +#ifdef ICONV_OPTION +extern iconv_t ic_send, ic_recv; +#endif const char phase_unknown[] = "unknown"; int ignore_timeout = 0; @@ -86,6 +89,7 @@ static int active_filecnt = 0; static OFF_T active_bytecnt = 0; static void read_loop(int fd, char *buf, size_t len); +static void mplex_write(enum msgcode code, char *buf, size_t len, int convert); struct flist_ndx_item { struct flist_ndx_item *next; @@ -100,7 +104,7 @@ static struct flist_ndx_list redo_list, struct msg_list_item { struct msg_list_item *next; - int len; + char convert; char buf[1]; }; @@ -204,7 +208,7 @@ void set_msg_fd_out(int fd) } /* Add a message to the pending MSG_* list. */ -static void msg_list_add(struct msg_list *lst, int code, char *buf, int len) +static void msg_list_add(struct msg_list *lst, int code, char *buf, int len, int convert) { struct msg_list_item *m; int sz = len + 4 + sizeof m[0] - 1; @@ -212,7 +216,7 @@ static void msg_list_add(struct msg_list if (!(m = (struct msg_list_item *)new_array(char, sz))) out_of_memory("msg_list_add"); m->next = NULL; - m->len = len + 4; + m->convert = convert; SIVAL(m->buf, 0, ((code+MPLEX_BASE)<<24) | len); memcpy(m->buf + 4, buf, len); if (lst->tail) @@ -267,7 +271,7 @@ static void read_msg_fd(void) exit_cleanup(RERR_STREAMIO); } read_loop(fd, buf, len); - send_msg(MSG_DELETED, buf, len); + send_msg(MSG_DELETED, buf, len, 1); break; case MSG_SUCCESS: if (len != 4 || !am_generator) { @@ -277,7 +281,7 @@ static void read_msg_fd(void) read_loop(fd, buf, len); if (remove_source_files) { decrement_active_files(IVAL(buf,0)); - send_msg(MSG_SUCCESS, buf, len); + send_msg(MSG_SUCCESS, buf, len, 0); } if (preserve_hard_links) flist_ndx_push(&hlink_list, IVAL(buf,0)); @@ -297,7 +301,7 @@ static void read_msg_fd(void) if (n >= sizeof buf) n = sizeof buf - 1; read_loop(fd, buf, n); - rwrite(tag, buf, n); + rwrite(tag, buf, n, !am_generator); len -= n; } break; @@ -346,7 +350,8 @@ static int msg2genr_flush(int flush_it_a while (msg2genr.head) { struct msg_list_item *m = msg2genr.head; - int n = write(msg_fd_out, m->buf + written, m->len - written); + int len = (IVAL(m->buf, 0) & 0xFFFFFF) + 4; + int n = write(msg_fd_out, m->buf + written, len - written); if (n < 0) { if (errno == EINTR) continue; @@ -360,7 +365,7 @@ static int msg2genr_flush(int flush_it_a tv.tv_usec = 0; if (!select(msg_fd_out+1, NULL, &fds, NULL, &tv)) check_timeout(); - } else if ((written += n) == m->len) { + } else if ((written += n) == len) { msg2genr.head = m->next; if (!msg2genr.head) msg2genr.tail = NULL; @@ -371,17 +376,17 @@ static int msg2genr_flush(int flush_it_a return 1; } -int send_msg(enum msgcode code, char *buf, int len) +int send_msg(enum msgcode code, char *buf, int len, int convert) { if (msg_fd_out < 0) { if (!defer_forwarding_messages) - return io_multiplex_write(code, buf, len); + return io_multiplex_write(code, buf, len, convert); if (!io_multiplexing_out) return 0; - msg_list_add(&msg2sndr, code, buf, len); + msg_list_add(&msg2sndr, code, buf, len, convert); return 1; } - msg_list_add(&msg2genr, code, buf, len); + msg_list_add(&msg2genr, code, buf, len, convert); msg2genr_flush(NORMAL_FLUSH); return 1; } @@ -788,7 +793,30 @@ static int readfd_unbuffered(int fd, cha case MSG_DELETED: if (msg_bytes >= sizeof line) goto overflow; - read_loop(fd, line, msg_bytes); +#ifdef ICONV_OPTION + if (ic_recv != (iconv_t)-1) { + char buf[1024], *ibuf, *obuf = line; + size_t icnt, ocnt = sizeof line - 1; + int add_null = 0; + iconv(ic_send, NULL, 0, NULL, 0); + while (msg_bytes) { + icnt = msg_bytes > sizeof buf + ? sizeof buf : msg_bytes; + read_loop(fd, buf, icnt); + ibuf = buf; + if (!(msg_bytes -= icnt) && !buf[icnt-1]) + icnt--, add_null = 1; + if (iconv(ic_send, &ibuf,&icnt, + &obuf,&ocnt) == (size_t)-1) { + goto overflow; // XXX + } + } + if (add_null) + *obuf++ = '\0'; + msg_bytes = obuf - line; + } else +#endif + read_loop(fd, line, msg_bytes); /* A directory name was sent with the trailing null */ if (msg_bytes > 0 && !line[msg_bytes-1]) log_delete(line, S_IFDIR); @@ -816,7 +844,7 @@ static int readfd_unbuffered(int fd, cha exit_cleanup(RERR_STREAMIO); } read_loop(fd, line, msg_bytes); - rwrite((enum logcode)tag, line, msg_bytes); + rwrite((enum logcode)tag, line, msg_bytes, 1); break; default: rprintf(FERROR, "unexpected tag %d [%s]\n", @@ -1143,11 +1171,13 @@ static void msg2sndr_flush(void) while (msg2sndr.head && io_multiplexing_out) { struct msg_list_item *m = msg2sndr.head; + int len = IVAL(m->buf, 0) & 0xFFFFFF; + int tag = *((uchar*)m->buf+3) - MPLEX_BASE; if (!(msg2sndr.head = m->next)) msg2sndr.tail = NULL; - stats.total_written += m->len; + stats.total_written += len + 4; defer_forwarding_messages = 1; - writefd_unbuffered(sock_f_out, m->buf, m->len); + mplex_write(tag, m->buf + 4, len, m->convert); defer_forwarding_messages = 0; free(m); } @@ -1157,14 +1187,14 @@ static void msg2sndr_flush(void) * Write an message to a multiplexed stream. If this fails then rsync * exits. **/ -static void mplex_write(enum msgcode code, char *buf, size_t len) +static void mplex_write(enum msgcode code, char *buf, size_t len, int convert) { - char buffer[1024]; + char buffer[BIGPATHBUFLEN]; /* Oversized for use by iconv code. */ size_t n = len; SIVAL(buffer, 0, ((MPLEX_BASE + (int)code)<<24) + len); - if (n > sizeof buffer - 4) + if (convert || n > 1024 - 4) /* We know that BIGPATHBUFLEN > 1024 */ n = 0; else memcpy(buffer + 4, buf, n); @@ -1174,6 +1204,28 @@ static void mplex_write(enum msgcode cod len -= n; buf += n; +#ifdef ICONV_OPTION + if (convert) { + iconv(ic_send, NULL, 0, NULL, 0); + defer_forwarding_messages = 1; + while (len) { + char *obuf = buffer; + size_t ocnt = sizeof buffer; + while (iconv(ic_send, &buf,&len, + &obuf,&ocnt) == (size_t)-1) { + assert(len > 0); + if (errno == E2BIG || !ocnt) + break; + *obuf++ = *buf++; + ocnt--, len--; + } + n = obuf - buffer; + writefd_unbuffered(sock_f_out, buffer, n); + } + defer_forwarding_messages = 0; + msg2sndr_flush(); + } else +#endif if (len) { defer_forwarding_messages = 1; writefd_unbuffered(sock_f_out, buf, len); @@ -1191,7 +1243,7 @@ void io_flush(int flush_it_all) return; if (io_multiplexing_out) - mplex_write(MSG_DATA, iobuf_out, iobuf_out_cnt); + mplex_write(MSG_DATA, iobuf_out, iobuf_out_cnt, 0); else writefd_unbuffered(sock_f_out, iobuf_out, iobuf_out_cnt); iobuf_out_cnt = 0; @@ -1370,14 +1422,18 @@ void io_start_multiplex_in(void) } /** Write an message to the multiplexed data stream. */ -int io_multiplex_write(enum msgcode code, char *buf, size_t len) +int io_multiplex_write(enum msgcode code, char *buf, size_t len, int convert) { if (!io_multiplexing_out) return 0; io_flush(NORMAL_FLUSH); stats.total_written += (len+4); - mplex_write(code, buf, len); +#ifdef ICONV_OPTION + if (ic_send == (iconv_t)-1) +#endif + convert = 0; + mplex_write(code, buf, len, convert); return 1; } --- old/log.c +++ new/log.c @@ -21,15 +21,13 @@ */ #include "rsync.h" -#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H -#include -#endif extern int verbose; extern int dry_run; extern int am_daemon; extern int am_server; extern int am_sender; +extern int am_generator; extern int local_server; extern int quiet; extern int module_id; @@ -49,6 +47,9 @@ extern char *logfile_name; #if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H extern iconv_t ic_chck; #endif +#ifdef ICONV_OPTION +extern iconv_t ic_send, ic_recv; +#endif extern char curr_dir[]; extern unsigned int module_dirlen; @@ -233,17 +234,25 @@ static void filtered_fwrite(FILE *f, con /* this is the underlying (unformatted) rsync debugging function. Call * it with FINFO, FERROR or FLOG. Note: recursion can happen with * certain fatal conditions. */ -void rwrite(enum logcode code, char *buf, int len) +void rwrite(enum logcode code, char *buf, int len, int is_utf8) { int trailing_CR_or_NL; FILE *f = NULL; +#ifdef ICONV_OPTION + iconv_t ic = is_utf8 && ic_recv != (iconv_t)-1 ? ic_recv : ic_chck; +#else +#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H + iconv_t ic = ic_chck; +#endif +#endif if (len < 0) exit_cleanup(RERR_MESSAGEIO); if (am_server && msg_fd_out >= 0) { + assert(!is_utf8); /* Pass the message to our sibling. */ - send_msg((enum msgcode)code, buf, len); + send_msg((enum msgcode)code, buf, len, 0); return; } @@ -276,7 +285,7 @@ void rwrite(enum logcode code, char *buf if (am_server) { /* Pass the message to the non-server side. */ - if (send_msg((enum msgcode)code, buf, len)) + if (send_msg((enum msgcode)code, buf, len, !is_utf8)) return; if (am_daemon) { /* TODO: can we send the error to the user somehow? */ @@ -300,13 +309,13 @@ void rwrite(enum logcode code, char *buf ? buf[--len] : 0; #if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H - if (ic_chck != (iconv_t)-1) { + if (ic != (iconv_t)-1) { char convbuf[1024]; char *in_buf = buf, *out_buf = convbuf; size_t in_cnt = len, out_cnt = sizeof convbuf - 1; - iconv(ic_chck, NULL, 0, NULL, 0); - while (iconv(ic_chck, &in_buf,&in_cnt, + iconv(ic, NULL, 0, NULL, 0); + while (iconv(ic, &in_buf,&in_cnt, &out_buf,&out_cnt) == (size_t)-1) { if (out_buf != convbuf) { filtered_fwrite(f, convbuf, out_buf - convbuf, 0); @@ -368,7 +377,7 @@ void rprintf(enum logcode code, const ch } } - rwrite(code, buf, len); + rwrite(code, buf, len, 0); } /* This is like rprintf, but it also tries to print some @@ -399,7 +408,7 @@ void rsyserr(enum logcode code, int errc if (len >= sizeof buf) exit_cleanup(RERR_MESSAGEIO); - rwrite(code, buf, len); + rwrite(code, buf, len, 0); } void rflush(enum logcode code) @@ -662,7 +671,7 @@ static void log_formatted(enum logcode c p = s + len; } - rwrite(code, buf, total); + rwrite(code, buf, total, 0); } /* Return 1 if the format escape is in the log-format string (e.g. look for @@ -737,7 +746,7 @@ void log_delete(char *fname, int mode) else if (am_server && protocol_version >= 29 && len < MAXPATHLEN) { if (S_ISDIR(mode)) len++; /* directories include trailing null */ - send_msg(MSG_DELETED, fname, len); + send_msg(MSG_DELETED, fname, len, am_generator); } else { fmt = stdout_format_has_o_or_i ? stdout_format : "deleting %n"; log_formatted(FCLIENT, fmt, "del.", &file, &stats, --- old/main.c +++ new/main.c @@ -724,7 +724,7 @@ static int do_recv(int f_in,int f_out,st io_flush(FULL_FLUSH); handle_stats(f_in); - send_msg(MSG_DONE, "", 0); + send_msg(MSG_DONE, "", 0, 0); io_flush(FULL_FLUSH); /* Handle any keep-alive packets from the post-processing work --- old/options.c +++ new/options.c @@ -176,6 +176,11 @@ int list_only = 0; #define MAX_BATCH_NAME_LEN 256 /* Must be less than MAXPATHLEN-13 */ char *batch_name = NULL; +#ifdef ICONV_OPTION +int need_unsorted_flist = 0; +char *iconv_opt = ICONV_OPTION; +#endif + struct chmod_mode_struct *chmod_modes = NULL; static int daemon_opt; /* sets am_daemon after option error-reporting */ @@ -200,6 +205,7 @@ static void print_rsync_version(enum log char const *have_inplace = "no "; char const *hardlinks = "no "; char const *links = "no "; + char const *iconv = "no "; char const *ipv6 = "no "; STRUCT_STAT *dumstat; @@ -219,6 +225,10 @@ static void print_rsync_version(enum log links = ""; #endif +#ifdef ICONV_OPTION + iconv = ""; +#endif + #ifdef INET6 ipv6 = ""; #endif @@ -235,9 +245,9 @@ static void print_rsync_version(enum log /* Note that this field may not have type ino_t. It depends * on the complicated interaction between largefile feature * macros. */ - rprintf(f, " %sinplace, %sIPv6, " + rprintf(f, " %sinplace, %sIPv6, %siconv, " "%d-bit system inums, %d-bit internal inums\n", - have_inplace, ipv6, + have_inplace, ipv6, iconv, (int) (sizeof dumstat->st_ino * 8), (int) (sizeof (int64) * 8)); #ifdef MAINTAINER_MODE @@ -381,6 +391,9 @@ void usage(enum logcode F) rprintf(F," --only-write-batch=FILE like --write-batch but w/o updating destination\n"); rprintf(F," --read-batch=FILE read a batched update from FILE\n"); rprintf(F," --protocol=NUM force an older protocol version to be used\n"); +#ifdef ICONV_OPTION + rprintf(F," --iconv=CONVERT_SPEC request charset conversion of filesnames\n"); +#endif #ifdef INET6 rprintf(F," -4, --ipv4 prefer IPv4\n"); rprintf(F," -6, --ipv6 prefer IPv6\n"); @@ -530,6 +543,9 @@ static struct poptOption long_options[] {"rsh", 'e', POPT_ARG_STRING, &shell_cmd, 0, 0, 0 }, {"rsync-path", 0, POPT_ARG_STRING, &rsync_path, 0, 0, 0 }, {"temp-dir", 'T', POPT_ARG_STRING, &tmpdir, 0, 0, 0 }, +#ifdef ICONV_OPTION + {"iconv", 0, POPT_ARG_STRING, &iconv_opt, 0, 0, 0 }, +#endif #ifdef INET6 {"ipv4", '4', POPT_ARG_VAL, &default_af_hint, AF_INET, 0, 0 }, {"ipv6", '6', POPT_ARG_VAL, &default_af_hint, AF_INET6, 0, 0 }, @@ -800,6 +816,11 @@ int parse_arguments(int *argc, const cha if (am_daemon) set_refuse_options("log-file*"); +#ifdef ICONV_OPTION + if (!am_daemon && (arg = getenv("RSYNC_ICONV")) != NULL && *arg) + iconv_opt = strdup(arg); +#endif + /* TODO: Call poptReadDefaultConfig; handle errors. */ /* The context leaks in case of an error, but if there's a @@ -825,6 +846,9 @@ int parse_arguments(int *argc, const cha long_options, 0); am_server = 1; } +#ifdef ICONV_OPTION + iconv_opt = NULL; +#endif break; case OPT_SENDER: @@ -842,6 +866,9 @@ int parse_arguments(int *argc, const cha sizeof err_buf); return 0; } +#ifdef ICONV_OPTION + iconv_opt = NULL; +#endif poptFreeContext(pc); pc = poptGetContext(RSYNC_NAME, *argc, *argv, long_daemon_options, 0); @@ -1110,6 +1137,15 @@ int parse_arguments(int *argc, const cha exit_cleanup(0); } +#ifdef ICONV_OPTION + if (iconv_opt) { + if (!am_server && strcmp(iconv_opt, "-") == 0) + iconv_opt = NULL; + else + need_unsorted_flist = 1; + } +#endif + #ifndef SUPPORT_LINKS if (preserve_links && !am_sender) { snprintf(err_buf, sizeof err_buf, @@ -1608,6 +1644,19 @@ void server_options(char **args,int *arg args[ac++] = "--log-format=X"; } +#ifdef ICONV_OPTION + if (iconv_opt) { + char *set = strchr(iconv_opt, ','); + if (set) + set++; + else + set = iconv_opt; + if (asprintf(&arg, "--iconv=%s", set) < 0) + goto oom; + args[ac++] = arg; + } +#endif + if (block_size) { if (asprintf(&arg, "-B%lu", block_size) < 0) goto oom; --- old/receiver.c +++ new/receiver.c @@ -303,7 +303,7 @@ static void handle_delayed_updates(struc || (preserve_hard_links && file->link_u.links)) { SIVAL(numbuf, 0, i); - send_msg(MSG_SUCCESS,numbuf,4); + send_msg(MSG_SUCCESS,numbuf,4,0); } handle_partial_dir(partialptr, PDIR_DELETE); } @@ -382,7 +382,7 @@ int recv_files(int f_in, struct file_lis rprintf(FINFO, "recv_files phase=%d\n", phase); if (phase == 2 && delay_updates) handle_delayed_updates(flist, local_name); - send_msg(MSG_DONE, "", 0); + send_msg(MSG_DONE, "", 0, 0); if (keep_partial && !partial_dir) make_backups = 0; /* prevents double backup */ if (append_mode) { @@ -657,7 +657,7 @@ int recv_files(int f_in, struct file_lis if (remove_source_files || (preserve_hard_links && file->link_u.links)) { SIVAL(numbuf, 0, i); - send_msg(MSG_SUCCESS, numbuf, 4); + send_msg(MSG_SUCCESS, numbuf, 4, 0); } } else if (!recv_ok) { int msgtype = phase || read_batch ? FERROR : FINFO; @@ -682,7 +682,7 @@ int recv_files(int f_in, struct file_lis } if (!phase) { SIVAL(numbuf, 0, i); - send_msg(MSG_REDO, numbuf, 4); + send_msg(MSG_REDO, numbuf, 4, 0); } } } --- old/rsync.c +++ new/rsync.c @@ -21,9 +21,6 @@ */ #include "rsync.h" -#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H -#include -#endif #if defined HAVE_LIBCHARSET_H && defined HAVE_LOCALE_CHARSET #include #elif defined HAVE_LANGINFO_H && defined HAVE_NL_LANGINFO @@ -48,11 +45,17 @@ extern int inplace; extern int keep_dirlinks; extern int make_backups; extern mode_t orig_umask; +#ifdef ICONV_OPTION +extern char *iconv_opt; +#endif extern struct stats stats; extern struct chmod_mode_struct *daemon_chmod_modes; #if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H iconv_t ic_chck = (iconv_t)-1; +#ifdef ICONV_OPTION +iconv_t ic_send = (iconv_t)-1, ic_recv = (iconv_t)-1; +#endif static const char *default_charset(void) { @@ -67,8 +70,10 @@ static const char *default_charset(void) void setup_iconv() { + const char *charset, *defset = default_charset(); + char *cp; + if (!am_server && !allow_8bit_chars) { - const char *defset = default_charset(); /* It's OK if this fails... */ ic_chck = iconv_open(defset, defset); @@ -86,6 +91,41 @@ void setup_iconv() } } } + +# ifdef ICONV_OPTION + if (!iconv_opt) + return; + + if ((cp = strchr(iconv_opt, ',')) != NULL) { + if (am_server) /* A local transfer needs this. */ + iconv_opt = cp + 1; + else + *cp = '\0'; + } + + if (!*iconv_opt || (*iconv_opt == '.' && iconv_opt[1] == '\0')) + charset = defset; + else + charset = iconv_opt; + + if ((ic_send = iconv_open(UTF8_CHARSET, charset)) == (iconv_t)-1) { + rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n", + UTF8_CHARSET, charset); + exit_cleanup(RERR_UNSUPPORTED); + } + + if ((ic_recv = iconv_open(charset, UTF8_CHARSET)) == (iconv_t)-1) { + rprintf(FERROR, "iconv_open(\"%s\", \"%s\") failed\n", + charset, UTF8_CHARSET); + exit_cleanup(RERR_UNSUPPORTED); + } + + if (verbose > 1) { + rprintf(FINFO, "%s charset: %s\n", + am_server ? "server" : "client", + *charset ? charset : "[LOCALE]"); + } +# endif } #endif --- old/rsync.h +++ new/rsync.h @@ -328,6 +328,10 @@ enum msgcode { # include #endif +#if defined HAVE_ICONV_OPEN && defined HAVE_ICONV_H +#include +#endif + #include #include "lib/pool_alloc.h" --- old/rsync.yo +++ new/rsync.yo @@ -398,6 +398,7 @@ to the detailed description below for a --only-write-batch=FILE like --write-batch but w/o updating dest --read-batch=FILE read a batched update from FILE --protocol=NUM force an older protocol version to be used + --iconv=CONVERT_SPEC request charset conversion of filesnames --checksum-seed=NUM set block/file checksum seed (advanced) -4, --ipv4 prefer IPv4 -6, --ipv6 prefer IPv6 @@ -1767,6 +1768,15 @@ bf(--read-batch) option, you should use batch file to force the older protocol version to be used in the batch file (assuming you can't upgrade the rsync on the reading system). +dit(bf(--iconv=CONVERT_SPEC)) Rsync can convert filenames between character +sets using this option. Using a CONVERT_SPEC of "." tells rsync to look up +the default character-set via the locale setting. Alternately, you can +fully specify what conversion to do by giving a local and a remote charset +separated by a comma (local first), e.g. bf(--iconv=utf8,iso88591). +Finally, you can specify a CONVERT_SPEC of "-" to turn off any conversion. +The default setting of this option is site-specific, and can also be +affected via the RSYNC_ICONV environment variable. + dit(bf(-4, --ipv4) or bf(-6, --ipv6)) Tells rsync to prefer IPv4/IPv6 when creating sockets. This only affects sockets that rsync has direct control over, such as the outgoing socket when directly contacting an @@ -2513,6 +2523,8 @@ startdit() dit(bf(CVSIGNORE)) The CVSIGNORE environment variable supplements any ignore patterns in .cvsignore files. See the bf(--cvs-exclude) option for more details. +dit(bf(RSYNC_ICONV)) Specify a default bf(--iconv) setting using this +environment variable. dit(bf(RSYNC_RSH)) The RSYNC_RSH environment variable allows you to override the default shell used as the transport for rsync. Command line options are permitted after the command name, just as in the bf(-e) option. --- old/socket.c +++ new/socket.c @@ -410,7 +410,7 @@ static int *open_socket_in(int type, int * unsuccessful, or if the daemon is being run with -vv. */ for (s = 0; s < ecnt; s++) { if (!i || verbose > 1) - rwrite(FLOG, errmsgs[s], strlen(errmsgs[s])); + rwrite(FLOG, errmsgs[s], strlen(errmsgs[s]), 0); free(errmsgs[s]); } free(errmsgs); --- old/testsuite/rsync.fns +++ new/testsuite/rsync.fns @@ -56,6 +56,8 @@ filter_outfile() { -e '/^done$/d' \ -e '/ --whole-file$/d' \ -e '/^total: /d' \ + -e '/^client charset: /d' \ + -e '/^server charset: /d' \ -e '/^$/,$d' \ <"$outfile" >"$outfile.new" mv "$outfile.new" "$outfile"