From 9ed1a1bfff616f19973f1205d0b6336ba8a3f472 Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 16:58:33 -0400 Subject: [PATCH 1/7] added support for skipping submodules --- _functions.sh | 131 ++++++++++++++++++++++++++------------------------ settings.ini | 2 + 2 files changed, 69 insertions(+), 64 deletions(-) diff --git a/_functions.sh b/_functions.sh index 6304799..0d5b874 100644 --- a/_functions.sh +++ b/_functions.sh @@ -203,78 +203,81 @@ function _discover_submodules() unset IGNORE_DIRS SUBMODULES ACTIONS SELECTION SUBDIRS choices options MENU FLAGS _get_svn_layout clear - echo "Discovering potential submodule candidates..." - # Get the potential list of submodules, with branches, tags and trunk - svn -R list ${REPO_URL} ${SVN_OPTIONS}|grep -E '(/trunk/$|/branches/$|/tags/$)' > /tmp/submodules.txt - # it turns out, some folks have .git in their repos, and this falsely - # identifies those as submodules. Let's remove those entries and not - # present the user with the option to migrate them - sed -i 's/\/.git\//d' /tmp/submodules.txt - # Remove empty "trunk", "tags" and "branches" from the list of potentials - for DIR in $(cat /tmp/submodules.txt); - do - FILES=$(svn list ${REPO_URL}/${DIR} ${SVN_OPTIONS}) - if [[ ${#FILES} -le 1 ]] - then - sed -i "s/${DIR}/d" /tmp/submodules.txt - fi - done - # Get the path to the submodules - export SUBMODULES=$(grep -E '(/trunk/$|/branches/$|/tags/$)' /tmp/submodules.txt|\ - sed -e 's/trunk\/$//' -e 's/tags\/$//' -e 's/branches\/$//'|sort|uniq) - - # Print a report of the discovered submodules - if [[ ${#SUBMODULES} -le 4 ]] + if [[ ${ENABLE_SUBMODULES} ]] then - echo "There were no nested repositories discovered" - else - options=($(echo ${SUBMODULES})) - #Actions to take based on selection - function ACTIONS { - for NUM in ${!options[@]}; do - [[ ${choices[NUM]} ]] && SUBDIRS+="${options[NUM]} " - done - if [[ ! -z ${SUBDIRS} ]] + echo "Discovering potential submodule candidates..." + # Get the potential list of submodules, with branches, tags and trunk + svn -R list ${REPO_URL} ${SVN_OPTIONS}|grep -E '(/trunk/$|/branches/$|/tags/$)' > /tmp/submodules.txt + # it turns out, some folks have .git in their repos, and this falsely + # identifies those as submodules. Let's remove those entries and not + # present the user with the option to migrate them + sed -i 's/\/.git\//d' /tmp/submodules.txt + # Remove empty "trunk", "tags" and "branches" from the list of potentials + for DIR in $(cat /tmp/submodules.txt); + do + FILES=$(svn list ${REPO_URL}/${DIR} ${SVN_OPTIONS}) + if [[ ${#FILES} -le 1 ]] then - export IGNORE_DIRS=$(echo "'^("${SUBDIRS}")$'"|sed -e 's/ /|/g;s/\/|)/\/)/') - export FLAGS+=" --ignore-paths ${IGNORE_DIRS}" + sed -i "s/${DIR}/d" /tmp/submodules.txt fi - } - #Variables - ERROR=" " - #Clear screen for menu - clear - #Menu function - function MENU { - _print_banner "We have discovered the following folders that contain" \ - "branches, tags, or trunk. This typically means that teams are using" \ - "them as separate repositories, but there is no real method of" \ - "discovering this, outside of the 'svn:externals' property, which is" \ - "often not used. Please review the following folders and select which" \ - "ones are to be treated as git submodules" - echo "" - echo "Discovered Folders" + done + # Get the path to the submodules + export SUBMODULES=$(grep -E '(/trunk/$|/branches/$|/tags/$)' /tmp/submodules.txt|\ + sed -e 's/trunk\/$//' -e 's/tags\/$//' -e 's/branches\/$//'|sort|uniq) + + # Print a report of the discovered submodules + if [[ ${#SUBMODULES} -le 4 ]] + then + echo "There were no nested repositories discovered" + else + options=($(echo ${SUBMODULES})) + #Actions to take based on selection + function ACTIONS { for NUM in ${!options[@]}; do - echo "[""${choices[NUM]:- }""]" $(( NUM+1 ))") ${options[NUM]}" + [[ ${choices[NUM]} ]] && SUBDIRS+="${options[NUM]} " done - echo "$ERROR" - } - #Menu loop - while MENU && read -e -p "Select the desired options using their number (again to uncheck, ENTER when done): " -n1 SELECTION && [[ -n "${SELECTION}" ]]; do + if [[ ! -z ${SUBDIRS} ]] + then + export IGNORE_DIRS=$(echo "'^("${SUBDIRS}")$'"|sed -e 's/ /|/g;s/\/|)/\/)/') + export FLAGS+=" --ignore-paths ${IGNORE_DIRS}" + fi + } + #Variables + ERROR=" " + #Clear screen for menu clear - if [[ "${SELECTION}" == *[[:digit:]]* && ${SELECTION} -ge 1 && ${SELECTION} -le ${#options[@]} ]]; then - (( SELECTION-- )) - if [[ "${choices[SELECTION]}" == "+" ]]; then - choices[SELECTION]="" + #Menu function + function MENU { + _print_banner "We have discovered the following folders that contain" \ + "branches, tags, or trunk. This typically means that teams are using" \ + "them as separate repositories, but there is no real method of" \ + "discovering this, outside of the 'svn:externals' property, which is" \ + "often not used. Please review the following folders and select which" \ + "ones are to be treated as git submodules" + echo "" + echo "Discovered Folders" + for NUM in ${!options[@]}; do + echo "[""${choices[NUM]:- }""]" $(( NUM+1 ))") ${options[NUM]}" + done + echo "$ERROR" + } + #Menu loop + while MENU && read -e -p "Select the desired options using their number (again to uncheck, ENTER when done): " -n1 SELECTION && [[ -n "${SELECTION}" ]]; do + clear + if [[ "${SELECTION}" == *[[:digit:]]* && ${SELECTION} -ge 1 && ${SELECTION} -le ${#options[@]} ]]; then + (( SELECTION-- )) + if [[ "${choices[SELECTION]}" == "+" ]]; then + choices[SELECTION]="" + else + choices[SELECTION]="+" + fi + ERROR=" " else - choices[SELECTION]="+" + ERROR="Invalid option: ${SELECTION}" fi - ERROR=" " - else - ERROR="Invalid option: ${SELECTION}" - fi - done - ACTIONS + done + ACTIONS + fi fi } diff --git a/settings.ini b/settings.ini index 79a42cf..4dc3148 100644 --- a/settings.ini +++ b/settings.ini @@ -16,3 +16,5 @@ SVN_PASSWORD=anonymous CONVERT_HISTORY=true ## Max file size for the repo. Default=100MB MAX_FILE_SIZE=100 +## Find and migrate nested repositories? +ENABLE_SUBMODULES=false From 14d1385e1d19bb471751820431b7822f8eb6ecd1 Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 17:03:14 -0400 Subject: [PATCH 2/7] updated logic to the main script instead of functions --- _functions.sh | 131 ++++++++++++++++++++++++-------------------------- svn2github.sh | 9 +++- 2 files changed, 71 insertions(+), 69 deletions(-) diff --git a/_functions.sh b/_functions.sh index 0d5b874..6304799 100644 --- a/_functions.sh +++ b/_functions.sh @@ -203,81 +203,78 @@ function _discover_submodules() unset IGNORE_DIRS SUBMODULES ACTIONS SELECTION SUBDIRS choices options MENU FLAGS _get_svn_layout clear - if [[ ${ENABLE_SUBMODULES} ]] + echo "Discovering potential submodule candidates..." + # Get the potential list of submodules, with branches, tags and trunk + svn -R list ${REPO_URL} ${SVN_OPTIONS}|grep -E '(/trunk/$|/branches/$|/tags/$)' > /tmp/submodules.txt + # it turns out, some folks have .git in their repos, and this falsely + # identifies those as submodules. Let's remove those entries and not + # present the user with the option to migrate them + sed -i 's/\/.git\//d' /tmp/submodules.txt + # Remove empty "trunk", "tags" and "branches" from the list of potentials + for DIR in $(cat /tmp/submodules.txt); + do + FILES=$(svn list ${REPO_URL}/${DIR} ${SVN_OPTIONS}) + if [[ ${#FILES} -le 1 ]] + then + sed -i "s/${DIR}/d" /tmp/submodules.txt + fi + done + # Get the path to the submodules + export SUBMODULES=$(grep -E '(/trunk/$|/branches/$|/tags/$)' /tmp/submodules.txt|\ + sed -e 's/trunk\/$//' -e 's/tags\/$//' -e 's/branches\/$//'|sort|uniq) + + # Print a report of the discovered submodules + if [[ ${#SUBMODULES} -le 4 ]] then - echo "Discovering potential submodule candidates..." - # Get the potential list of submodules, with branches, tags and trunk - svn -R list ${REPO_URL} ${SVN_OPTIONS}|grep -E '(/trunk/$|/branches/$|/tags/$)' > /tmp/submodules.txt - # it turns out, some folks have .git in their repos, and this falsely - # identifies those as submodules. Let's remove those entries and not - # present the user with the option to migrate them - sed -i 's/\/.git\//d' /tmp/submodules.txt - # Remove empty "trunk", "tags" and "branches" from the list of potentials - for DIR in $(cat /tmp/submodules.txt); - do - FILES=$(svn list ${REPO_URL}/${DIR} ${SVN_OPTIONS}) - if [[ ${#FILES} -le 1 ]] + echo "There were no nested repositories discovered" + else + options=($(echo ${SUBMODULES})) + #Actions to take based on selection + function ACTIONS { + for NUM in ${!options[@]}; do + [[ ${choices[NUM]} ]] && SUBDIRS+="${options[NUM]} " + done + if [[ ! -z ${SUBDIRS} ]] then - sed -i "s/${DIR}/d" /tmp/submodules.txt + export IGNORE_DIRS=$(echo "'^("${SUBDIRS}")$'"|sed -e 's/ /|/g;s/\/|)/\/)/') + export FLAGS+=" --ignore-paths ${IGNORE_DIRS}" fi - done - # Get the path to the submodules - export SUBMODULES=$(grep -E '(/trunk/$|/branches/$|/tags/$)' /tmp/submodules.txt|\ - sed -e 's/trunk\/$//' -e 's/tags\/$//' -e 's/branches\/$//'|sort|uniq) - - # Print a report of the discovered submodules - if [[ ${#SUBMODULES} -le 4 ]] - then - echo "There were no nested repositories discovered" - else - options=($(echo ${SUBMODULES})) - #Actions to take based on selection - function ACTIONS { + } + #Variables + ERROR=" " + #Clear screen for menu + clear + #Menu function + function MENU { + _print_banner "We have discovered the following folders that contain" \ + "branches, tags, or trunk. This typically means that teams are using" \ + "them as separate repositories, but there is no real method of" \ + "discovering this, outside of the 'svn:externals' property, which is" \ + "often not used. Please review the following folders and select which" \ + "ones are to be treated as git submodules" + echo "" + echo "Discovered Folders" for NUM in ${!options[@]}; do - [[ ${choices[NUM]} ]] && SUBDIRS+="${options[NUM]} " + echo "[""${choices[NUM]:- }""]" $(( NUM+1 ))") ${options[NUM]}" done - if [[ ! -z ${SUBDIRS} ]] - then - export IGNORE_DIRS=$(echo "'^("${SUBDIRS}")$'"|sed -e 's/ /|/g;s/\/|)/\/)/') - export FLAGS+=" --ignore-paths ${IGNORE_DIRS}" - fi - } - #Variables - ERROR=" " - #Clear screen for menu + echo "$ERROR" + } + #Menu loop + while MENU && read -e -p "Select the desired options using their number (again to uncheck, ENTER when done): " -n1 SELECTION && [[ -n "${SELECTION}" ]]; do clear - #Menu function - function MENU { - _print_banner "We have discovered the following folders that contain" \ - "branches, tags, or trunk. This typically means that teams are using" \ - "them as separate repositories, but there is no real method of" \ - "discovering this, outside of the 'svn:externals' property, which is" \ - "often not used. Please review the following folders and select which" \ - "ones are to be treated as git submodules" - echo "" - echo "Discovered Folders" - for NUM in ${!options[@]}; do - echo "[""${choices[NUM]:- }""]" $(( NUM+1 ))") ${options[NUM]}" - done - echo "$ERROR" - } - #Menu loop - while MENU && read -e -p "Select the desired options using their number (again to uncheck, ENTER when done): " -n1 SELECTION && [[ -n "${SELECTION}" ]]; do - clear - if [[ "${SELECTION}" == *[[:digit:]]* && ${SELECTION} -ge 1 && ${SELECTION} -le ${#options[@]} ]]; then - (( SELECTION-- )) - if [[ "${choices[SELECTION]}" == "+" ]]; then - choices[SELECTION]="" - else - choices[SELECTION]="+" - fi - ERROR=" " + if [[ "${SELECTION}" == *[[:digit:]]* && ${SELECTION} -ge 1 && ${SELECTION} -le ${#options[@]} ]]; then + (( SELECTION-- )) + if [[ "${choices[SELECTION]}" == "+" ]]; then + choices[SELECTION]="" else - ERROR="Invalid option: ${SELECTION}" + choices[SELECTION]="+" fi - done - ACTIONS - fi + ERROR=" " + else + ERROR="Invalid option: ${SELECTION}" + fi + done + ACTIONS fi } diff --git a/svn2github.sh b/svn2github.sh index 2e4418a..bb4940b 100644 --- a/svn2github.sh +++ b/svn2github.sh @@ -11,8 +11,13 @@ fi _welcome _setup _svn_sizer -_discover_submodules -_process_submodules +if [[ ${ENABLE_SUBMODULES} ]] +then + _discover_submodules + _process_submodules +else + _get_svn_layout +fi _git_svn_clone ( cd ${REPO_NAME} From 059b7938bfdf8dd6f636064e7910cd4a39a5210b Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 17:06:17 -0400 Subject: [PATCH 3/7] added comment --- svn2github.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/svn2github.sh b/svn2github.sh index bb4940b..64dbd42 100644 --- a/svn2github.sh +++ b/svn2github.sh @@ -11,6 +11,7 @@ fi _welcome _setup _svn_sizer +## Process submodules if [[ ${ENABLE_SUBMODULES} ]] then _discover_submodules @@ -24,7 +25,7 @@ _git_svn_clone git config http.sslVerify false _prepare_github _migrate_trunk - _add_git_submodules + [[ ${ENABLE_SUBMODULES} ]] && _add_git_submodules _migrate_tags _migrate_branches ) From d32b06b6be9767ea1fe40e603d230d9f173a8d77 Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 17:07:55 -0400 Subject: [PATCH 4/7] added logic for clean cutover --- svn2github.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/svn2github.sh b/svn2github.sh index 64dbd42..79d55f8 100644 --- a/svn2github.sh +++ b/svn2github.sh @@ -19,7 +19,14 @@ then else _get_svn_layout fi -_git_svn_clone +## Perform a clean cutover or migrate history +if [[ ${MIGRATE_HISTORY} ]] +then + _git_svn_clone +else + _clean_cutover +fi +## Migrate trunk, branches, tags, submodules ( cd ${REPO_NAME} git config http.sslVerify false From d88522048292e6b41e844257eee7a5cb4a84773b Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 17:25:12 -0400 Subject: [PATCH 5/7] added git lfs migrate... forgot that before --- _functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/_functions.sh b/_functions.sh index 6304799..b881de9 100644 --- a/_functions.sh +++ b/_functions.sh @@ -457,6 +457,7 @@ function _initialize_lfs() EXTENSION=$(echo ${FILE}|awk -F'.' {'print " *."$3'}) git lfs track ${EXTENSION} git add ${EXTENSION} + git lfs migrate --include="*.${EXTENSION}" done git add .gitattributes git commit -m "Initialized Git-LFS" From baa86c255e12990951b885bea30b0a5a05509179 Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 18:33:21 -0400 Subject: [PATCH 6/7] added new values to README sample --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9ac8f69..6d98d25 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,8 @@ GITHUB_ORG=GitHub-Demo AUTHORS_FILE=/tmp/authors.txt SVN_USERNAME=anonymous SVN_PASSWORD=anonymous +ENABLE_SUBMODULES=true +MIGRATE_HISTORY=true ``` ## Caveats From 2d1a0940951f951c15452694297027d220a7cad1 Mon Sep 17 00:00:00 2001 From: Jared Murrell Date: Tue, 26 Jun 2018 22:04:12 -0400 Subject: [PATCH 7/7] added cleanup for LFS --- _functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_functions.sh b/_functions.sh index b881de9..6e7fc4c 100644 --- a/_functions.sh +++ b/_functions.sh @@ -461,6 +461,8 @@ function _initialize_lfs() done git add .gitattributes git commit -m "Initialized Git-LFS" + git reflog expire --expire-unreachable=now --all + git gc --prune=now fi }