From 5908fb17f9a1338dec4915f4cc6a17ef9dc2f78d Mon Sep 17 00:00:00 2001 From: 502647092 Date: Tue, 25 Jul 2017 16:31:50 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=20HttpKit=20?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=20=E6=B7=BB=E5=8A=A0=20Gogs=20=E4=BB=93?= =?UTF-8?q?=E5=BA=93=E8=BF=81=E7=A7=BB=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 502647092 --- pom.xml | 12 +- .../java/pw/yumc/YumCore/kit/HttpKit.java | 230 ++++++++---------- .../YumCore/plugin/MigrateCodingToGogs.java | 213 ++++++++++++++++ 3 files changed, 328 insertions(+), 127 deletions(-) create mode 100644 src/test/java/pw/yumc/YumCore/plugin/MigrateCodingToGogs.java diff --git a/pom.xml b/pom.xml index 2a470c2..b9f29b9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 pw.yumc YumCore - 1.8 + 1.8.1 ${project.artifactId} @@ -130,5 +130,15 @@ worldedit-bukkit 6.1.5 + + com.alibaba + fastjson + 1.2.28 + + + org.projectlombok + lombok + 1.16.8 + \ No newline at end of file diff --git a/src/main/java/pw/yumc/YumCore/kit/HttpKit.java b/src/main/java/pw/yumc/YumCore/kit/HttpKit.java index 9bd1298..c54bd66 100644 --- a/src/main/java/pw/yumc/YumCore/kit/HttpKit.java +++ b/src/main/java/pw/yumc/YumCore/kit/HttpKit.java @@ -1,6 +1,11 @@ package pw.yumc.YumCore.kit; -import java.io.*; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; @@ -12,16 +17,33 @@ import java.security.cert.X509Certificate; import java.util.Map; import java.util.Map.Entry; -import javax.net.ssl.*; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import lombok.Builder; +import lombok.Cleanup; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.val; /** * HttpKit */ public class HttpKit { - private static String GET = "GET"; - - private static String POST = "POST"; + public enum HttpMethod { + GET, + POST, + PUT, + HEADER, + DELETE, + PATCH + } private static String CHARSET = "UTF-8"; @@ -39,7 +61,7 @@ public class HttpKit { * @return 网页HTML */ public static String get(String url) { - return get(url, null, null); + return get(url, null); } /** @@ -67,71 +89,11 @@ public class HttpKit { * @return 网页HTML */ public static String get(String url, Map queryParas, Map headers) { - HttpURLConnection conn = null; - try { - conn = getHttpConnection(buildUrlWithQueryString(url, queryParas), GET, headers); - conn.connect(); - return readResponseString(conn); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (conn != null) { - conn.disconnect(); - } - } + return send(HttpInfo.builder().url(url).method(HttpMethod.GET).query(queryParas).header(headers).build()); } /** - * Post 获取网页 - * - * @param url - * 网址 - * @param queryParas - * 参数 - * @param data - * 数据 - * @return 网页HTML - */ - public static String post(String url, Map queryParas, String data) { - return post(url, queryParas, data, null); - } - - /** - * Post 获取网页 - * - * @param url - * 网址 - * @param queryParas - * 参数 - * @param data - * 数据 - * @param headers - * 头信息 - * @return 网页HTML - */ - public static String post(String url, Map queryParas, String data, Map headers) { - HttpURLConnection conn = null; - try { - conn = getHttpConnection(buildUrlWithQueryString(url, queryParas), POST, headers); - conn.connect(); - - OutputStream out = conn.getOutputStream(); - out.write(data.getBytes(CHARSET)); - out.flush(); - out.close(); - - return readResponseString(conn); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (conn != null) { - conn.disconnect(); - } - } - } - - /** - * Get 方法获取HTML + * Post 方法获取HTML * * @param url * 网址 @@ -140,64 +102,94 @@ public class HttpKit { * @return 网页HTML */ public static String post(String url, String data) { - return post(url, null, data, null); + return post(url, data, null); } /** - * Get 方法获取HTML + * Post 方法获取HTML * * @param url * 网址 * @param data * 查询参数 - * @param headers + * @param header * 头信息 * @return 网页HTML */ - public static String post(String url, String data, Map headers) { - return post(url, null, data, headers); + public static String post(String url, String data, Map header) { + return send(HttpInfo.builder().url(url).data(data).method(HttpMethod.POST).header(header).build()); + } + + /** + * DELETE 方法 + * + * @param url + * 地址 + * @param header + * 头信息 + * @return + */ + public static String delete(String url, Map header) { + return send(HttpInfo.builder().url(url).method(HttpMethod.DELETE).header(header).build()); + } + + /** + * Post 获取网页 + * + * @param info + * 链接信息 + * @return 网页HTML + */ + @SneakyThrows + public static String send(HttpInfo info) { + @Cleanup + HttpURLConnection conn = getHttpConnection(buildUrlWithQueryString(info), info.getMethod().name(), info.getHeader()); + conn.connect(); + if (StrKit.notBlank(info.getData())) { + OutputStream out = conn.getOutputStream(); + out.write(info.getData().getBytes(CHARSET)); + out.flush(); + out.close(); + } + return readResponseString(conn); + } + + @Data + @Builder + public static class HttpInfo { + private String url; + private HttpMethod method; + private Map query; + private String data; + private Map header; } /** * 构建查询串为字符串 * - * @param url + * @param info * 网址 - * @param queryParas - * 参数 * @return 构建后的地址 */ - private static String buildUrlWithQueryString(String url, Map queryParas) { - if (queryParas == null || queryParas.isEmpty()) { return url; } - + private static String buildUrlWithQueryString(HttpInfo info) { + val url = info.getUrl(); + val query = info.getQuery(); + if (query == null || query.isEmpty()) { return url; } StringBuilder sb = new StringBuilder(url); - boolean isFirst; if (!url.contains("?")) { - isFirst = true; sb.append("?"); - } else { - isFirst = false; } - - for (Entry entry : queryParas.entrySet()) { - if (isFirst) { - isFirst = false; - } else { - sb.append("&"); - } - + query.entrySet().stream().filter(entry -> StrKit.isBlank(entry.getValue())).forEach(entry -> { String key = entry.getKey(); String value = entry.getValue(); - if (StrKit.notBlank(value)) { - try { - value = URLEncoder.encode(value, CHARSET); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } + try { + value = URLEncoder.encode(value, CHARSET); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); } sb.append(key).append("=").append(value); - } - return sb.toString(); + }); + return sb.toString().substring(0, sb.length() - 1); } /** @@ -247,15 +239,12 @@ public class HttpKit { * * @return 安全套接字工厂 */ + @SneakyThrows private static SSLSocketFactory initSSLSocketFactory() { - try { - TrustManager[] tm = { new HttpKit.TrustAnyTrustManager() }; - SSLContext sslContext = SSLContext.getInstance("TLS", "SunJSSE"); - sslContext.init(null, tm, new java.security.SecureRandom()); - return sslContext.getSocketFactory(); - } catch (Exception e) { - throw new RuntimeException(e); - } + TrustManager[] tm = { new HttpKit.TrustAnyTrustManager() }; + SSLContext sslContext = SSLContext.getInstance("TLS", "SunJSSE"); + sslContext.init(null, tm, new java.security.SecureRandom()); + return sslContext.getSocketFactory(); } /** @@ -265,28 +254,17 @@ public class HttpKit { * HTTP连接 * @return 字符串 */ + @SneakyThrows private static String readResponseString(HttpURLConnection conn) { StringBuilder sb = new StringBuilder(); - InputStream inputStream = null; - try { - inputStream = conn.getInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, CHARSET)); - String line; - while ((line = reader.readLine()) != null) { - sb.append(line).append("\n"); - } - return sb.toString(); - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - if (inputStream != null) { - try { - inputStream.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } + @Cleanup + InputStream inputStream = conn.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, CHARSET)); + String line; + while ((line = reader.readLine()) != null) { + sb.append(line).append("\n"); } + return sb.toString(); } /** diff --git a/src/test/java/pw/yumc/YumCore/plugin/MigrateCodingToGogs.java b/src/test/java/pw/yumc/YumCore/plugin/MigrateCodingToGogs.java new file mode 100644 index 0000000..a603390 --- /dev/null +++ b/src/test/java/pw/yumc/YumCore/plugin/MigrateCodingToGogs.java @@ -0,0 +1,213 @@ +package pw.yumc.YumCore.plugin; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +import lombok.AllArgsConstructor; +import lombok.Data; +import pw.yumc.YumCore.kit.HttpKit; + +/** + * Created with IntelliJ IDEA + * + * @author 喵♂呜 + * Created on 2017/7/25 10:04. + */ +public class MigrateCodingToGogs { + //private String site = "http://git.yumc.pw"; + private String site = "https://try.gogs.io"; + private String username = "xxxxxxxxx";// Gogs账号 + private String password = "xxxxxxxxx";// Gogs密码 + private Map gogs_header; + private String coding_token; + + /** + * 删除所有项目 + */ + public void clearProject() { + JSONObject userInfo = getUser(); + System.out.println("Gogs用户ID: " + userInfo.getString("id")); + updateAuthorization(username); + List list = getNameListFromGogs(); + clearList(userInfo.getString("username"), list); + } + + private void clearList(String u, List list) { + list.forEach(s -> delete("/repos/" + u + "/" + s)); + } + + /** + * 镜像所有项目 + */ + public void migrateProject() { + JSONObject userInfo = getUser(); + System.out.println("Gogs用户ID: " + userInfo.getString("id")); + updateAuthorization(username); + List list = getListFromCoding(userInfo.getString("id")); + List exist = getNameListFromGogs(); + list.removeIf(repoInfo -> exist.contains(repoInfo.getName())); + migrateList(list); + } + + /** + * 更新Basic为token + */ + private void updateAuthorization(String user) { + String token = getUserToken(user); + System.out.println("Gogs用户Toekn: " + token); + getHeader().put("Authorization", "token " + token); + } + + /** + * 获取或创建用户Token + * + * @return 用户Token + */ + private String getUserToken(String user) { + JSONArray tks = JSON.parseArray(get("/users/" + user + "/tokens")); + if (tks.isEmpty()) { + return JSON.parseObject(post("/users/" + user + "/tokens", "name=migrate")).getString("sha1"); + } else { + return tks.getJSONObject(0).getString("sha1"); + } + } + + public Map getHeader() { + if (gogs_header == null) { + gogs_header = new HashMap<>(); + gogs_header.put("Content-Type", " application/x-www-form-urlencoded"); + gogs_header.put("Authorization", "Basic " + new String(Base64.getEncoder().encode((username + ":" + password).getBytes()))); + } + return gogs_header; + } + + private JSONObject getUser() { + return JSON.parseObject(get("/user")); + } + + private List getNameListFromGogs() { + List exist = new ArrayList<>(); + JSONArray arr = JSON.parseArray(get("/user/repos")); + for (int i = 0; i < arr.size(); i++) { + JSONObject obj = arr.getJSONObject(i); + exist.add(obj.getString("name")); + } + return exist; + } + + public List getListFromCoding(String uid) { + Map coding_header = new HashMap<>(); + coding_header.put("Authorization", "token " + coding_token); + String json = HttpKit.get("https://coding.net/api/projects?page=1&pageSize=1000&type=all", null, coding_header); + JSONArray projs = JSON.parseObject(json).getJSONObject("data").getJSONArray("list"); + System.out.println("Coding仓库数量: " + projs.size()); + List list = new ArrayList<>(); + for (int i = 0; i < projs.size(); i++) { + JSONObject proj = projs.getJSONObject(i); + list.add(new RepoInfo(proj.getString("https_url"), proj.getString("name"), username, password, uid, !proj.getBooleanValue("is_public"), proj.getString("description"))); + } + return list; + } + + public void migrateList(List projs) { + long start = System.currentTimeMillis(); + ExecutorService executor = Executors.newFixedThreadPool(10); + for (RepoInfo repoInfo : projs) { + executor.execute(() -> migrate(repoInfo)); + migrate(repoInfo); + } + executor.shutdown(); + try { + boolean loop; + do { //等待所有任务完成 + loop = !executor.awaitTermination(2, TimeUnit.SECONDS); //阻塞，直到线程池里所有任务结束 + } while (loop); + } catch (InterruptedException ignored) { + } + System.out.println(String.format("迁移完成 耗时 %s 毫秒!", System.currentTimeMillis() - start)); + } + + public void migrate(RepoInfo info) { + String data = new StringBuilder() + .append("clone_addr=") + .append(eu(info.getUrl())) + .append("&auth_username=") + .append(eu(info.getUser())) + .append("&auth_password=") + .append(eu(info.getPass())) + .append("&uid=") + .append(info.getUid()) + .append("&repo_name=") + .append(eu(info.getName())) + .append("&private=") + .append(info.isPrivate() ? "on" : "off") + .append("&mirror=on&description=") + .append(eu(info.getDescription())) + .toString(); + try { + get("/repos/" + username + "/" + info.getName()); + System.out.println("仓库: " + info.getName() + " 已存在!"); + } catch (Exception ex) { + try { + post("/repos/migrate", data); + } catch (Exception ignored) { + while (ignored.getCause() != null) { + ignored = (Exception) ignored.getCause(); + } + System.out.println("仓库: " + info.getName() + " 异常: " + ignored.getClass().getName() + ":" + ignored.getMessage()); + // 忽略超时以及其他异常 + } + System.out.println("仓库: " + info.getName() + " 迁移完成!"); + } + } + + public String get(String url) { + String result = HttpKit.get(site + "/api/v1" + url, getHeader()); + System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + "\n[DEBUG] Result: " + result); + return result; + } + + public String delete(String url) { + String result = HttpKit.delete(site + "/api/v1" + url, getHeader()); + System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + "\n[DEBUG] Result: " + result); + return result; + } + + public String post(String url, String data) { + String result = HttpKit.post(site + "/api/v1" + url, data, getHeader()); + System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + " DATA: " + data + "\n[DEBUG] Result: " + result); + return result; + } + + public static String eu(String str) { + try { + return URLEncoder.encode(str, "UTF-8"); + } catch (UnsupportedEncodingException e) { + return ""; + } + } + + @Data + @AllArgsConstructor + public static class RepoInfo { + private String url; + private String name; + private String user; + private String pass; + private String uid; + private boolean isPrivate; + private String description; + } +}