502647092/YumCore
1
0
Fork 0
mirror of https://e.coding.net/circlecloud/YumCore.git synced 2024-11-21 01:38:51 +00:00
Code Issues Releases Activity

feat: 更新 HttpKit 工具 添加 Gogs 仓库迁移脚本

Browse Source 
Signed-off-by: 502647092 <admin@yumc.pw>
This commit is contained in:
502647092 2017-07-25 16:31:50 +08:00
parent e949e4677a
commit 5908fb17f9
3 changed files with 328 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>pw.yumc</groupId>
<artifactId>YumCore</artifactId>
<version>1.8</version>
<version>1.8.1</version>
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
@ -130,5 +130,15 @@
<artifactId>worldedit-bukkit</artifactId>
<version>6.1.5</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.28</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.8</version>
</dependency>
</dependencies>
</project>

230
src/main/java/pw/yumc/YumCore/kit/HttpKit.java
View File

@ -1,6 +1,11 @@
package pw.yumc.YumCore.kit;
import java.io.*;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
@ -12,16 +17,33 @@ import java.security.cert.X509Certificate;
import java.util.Map;
import java.util.Map.Entry;
import javax.net.ssl.*;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import lombok.Builder;
import lombok.Cleanup;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.val;
/**
* HttpKit
*/
public class HttpKit {
private static String GET = "GET";
private static String POST = "POST";
public enum HttpMethod {
GET,
POST,
PUT,
HEADER,
DELETE,
PATCH
}
private static String CHARSET = "UTF-8";
@ -39,7 +61,7 @@ public class HttpKit {
* @return 网页HTML
*/
public static String get(String url) {
return get(url, null, null);
return get(url, null);
}
/**
@ -67,71 +89,11 @@ public class HttpKit {
* @return 网页HTML
*/
public static String get(String url, Map<String, String> queryParas, Map<String, String> headers) {
HttpURLConnection conn = null;
try {
conn = getHttpConnection(buildUrlWithQueryString(url, queryParas), GET, headers);
conn.connect();
return readResponseString(conn);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (conn != null) {
conn.disconnect();
}
}
return send(HttpInfo.builder().url(url).method(HttpMethod.GET).query(queryParas).header(headers).build());
}
/**
* Post 获取网页
*
* @param url
* 网址
* @param queryParas
* 参数
* @param data
* 数据
* @return 网页HTML
*/
public static String post(String url, Map<String, String> queryParas, String data) {
return post(url, queryParas, data, null);
}
/**
* Post 获取网页
*
* @param url
* 网址
* @param queryParas
* 参数
* @param data
* 数据
* @param headers
* 头信息
* @return 网页HTML
*/
public static String post(String url, Map<String, String> queryParas, String data, Map<String, String> headers) {
HttpURLConnection conn = null;
try {
conn = getHttpConnection(buildUrlWithQueryString(url, queryParas), POST, headers);
conn.connect();
OutputStream out = conn.getOutputStream();
out.write(data.getBytes(CHARSET));
out.flush();
out.close();
return readResponseString(conn);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (conn != null) {
conn.disconnect();
}
}
}
/**
* Get 方法获取HTML
* Post 方法获取HTML
*
* @param url
* 网址
@ -140,64 +102,94 @@ public class HttpKit {
* @return 网页HTML
*/
public static String post(String url, String data) {
return post(url, null, data, null);
return post(url, data, null);
}
/**
* Get 方法获取HTML
* Post 方法获取HTML
*
* @param url
* 网址
* @param data
* 查询参数
* @param headers
* @param header
* 头信息
* @return 网页HTML
*/
public static String post(String url, String data, Map<String, String> headers) {
return post(url, null, data, headers);
public static String post(String url, String data, Map<String, String> header) {
return send(HttpInfo.builder().url(url).data(data).method(HttpMethod.POST).header(header).build());
}
/**
* DELETE 方法
*
* @param url
* 地址
* @param header
* 头信息
* @return
*/
public static String delete(String url, Map<String, String> header) {
return send(HttpInfo.builder().url(url).method(HttpMethod.DELETE).header(header).build());
}
/**
* Post 获取网页
*
* @param info
* 链接信息
* @return 网页HTML
*/
@SneakyThrows
public static String send(HttpInfo info) {
@Cleanup
HttpURLConnection conn = getHttpConnection(buildUrlWithQueryString(info), info.getMethod().name(), info.getHeader());
conn.connect();
if (StrKit.notBlank(info.getData())) {
OutputStream out = conn.getOutputStream();
out.write(info.getData().getBytes(CHARSET));
out.flush();
out.close();
}
return readResponseString(conn);
}
@Data
@Builder
public static class HttpInfo {
private String url;
private HttpMethod method;
private Map<String, String> query;
private String data;
private Map<String, String> header;
}
/**
* 构建查询串为字符串
*
* @param url
* @param info
* 网址
* @param queryParas
* 参数
* @return 构建后的地址
*/
private static String buildUrlWithQueryString(String url, Map<String, String> queryParas) {
if (queryParas == null || queryParas.isEmpty()) { return url; }
private static String buildUrlWithQueryString(HttpInfo info) {
val url = info.getUrl();
val query = info.getQuery();
if (query == null || query.isEmpty()) { return url; }
StringBuilder sb = new StringBuilder(url);
boolean isFirst;
if (!url.contains("?")) {
isFirst = true;
sb.append("?");
} else {
isFirst = false;
}
for (Entry<String, String> entry : queryParas.entrySet()) {
if (isFirst) {
isFirst = false;
} else {
sb.append("&");
}
query.entrySet().stream().filter(entry -> StrKit.isBlank(entry.getValue())).forEach(entry -> {
String key = entry.getKey();
String value = entry.getValue();
if (StrKit.notBlank(value)) {
try {
value = URLEncoder.encode(value, CHARSET);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
try {
value = URLEncoder.encode(value, CHARSET);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
sb.append(key).append("=").append(value);
}
return sb.toString();
});
return sb.toString().substring(0, sb.length() - 1);
}
/**
@ -247,15 +239,12 @@ public class HttpKit {
*
* @return 安全套接字工厂
*/
@SneakyThrows
private static SSLSocketFactory initSSLSocketFactory() {
try {
TrustManager[] tm = { new HttpKit.TrustAnyTrustManager() };
SSLContext sslContext = SSLContext.getInstance("TLS", "SunJSSE");
sslContext.init(null, tm, new java.security.SecureRandom());
return sslContext.getSocketFactory();
} catch (Exception e) {
throw new RuntimeException(e);
}
TrustManager[] tm = { new HttpKit.TrustAnyTrustManager() };
SSLContext sslContext = SSLContext.getInstance("TLS", "SunJSSE");
sslContext.init(null, tm, new java.security.SecureRandom());
return sslContext.getSocketFactory();
}
/**
@ -265,28 +254,17 @@ public class HttpKit {
* HTTP连接
* @return 字符串
*/
@SneakyThrows
private static String readResponseString(HttpURLConnection conn) {
StringBuilder sb = new StringBuilder();
InputStream inputStream = null;
try {
inputStream = conn.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, CHARSET));
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
return sb.toString();
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
e.printStackTrace();
}
}
@Cleanup
InputStream inputStream = conn.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, CHARSET));
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("\n");
}
return sb.toString();
}
/**

213
src/test/java/pw/yumc/YumCore/plugin/MigrateCodingToGogs.java Normal file
View File

@ -0,0 +1,213 @@
package pw.yumc.YumCore.plugin;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import lombok.AllArgsConstructor;
import lombok.Data;
import pw.yumc.YumCore.kit.HttpKit;
/**
* Created with IntelliJ IDEA
*
* @author 喵♂呜
* Created on 2017/7/25 10:04.
*/
public class MigrateCodingToGogs {
//private String site = "http://git.yumc.pw";
private String site = "https://try.gogs.io";
private String username = "xxxxxxxxx";// Gogs账号
private String password = "xxxxxxxxx";// Gogs密码
private Map<String, String> gogs_header;
private String coding_token;
/**
* 删除所有项目
*/
public void clearProject() {
JSONObject userInfo = getUser();
System.out.println("Gogs用户ID: " + userInfo.getString("id"));
updateAuthorization(username);
List<String> list = getNameListFromGogs();
clearList(userInfo.getString("username"), list);
}
private void clearList(String u, List<String> list) {
list.forEach(s -> delete("/repos/" + u + "/" + s));
}
/**
* 镜像所有项目
*/
public void migrateProject() {
JSONObject userInfo = getUser();
System.out.println("Gogs用户ID: " + userInfo.getString("id"));
updateAuthorization(username);
List<RepoInfo> list = getListFromCoding(userInfo.getString("id"));
List<String> exist = getNameListFromGogs();
list.removeIf(repoInfo -> exist.contains(repoInfo.getName()));
migrateList(list);
}
/**
* 更新Basic为token
*/
private void updateAuthorization(String user) {
String token = getUserToken(user);
System.out.println("Gogs用户Toekn: " + token);
getHeader().put("Authorization", "token " + token);
}
/**
* 获取或创建用户Token
*
* @return 用户Token
*/
private String getUserToken(String user) {
JSONArray tks = JSON.parseArray(get("/users/" + user + "/tokens"));
if (tks.isEmpty()) {
return JSON.parseObject(post("/users/" + user + "/tokens", "name=migrate")).getString("sha1");
} else {
return tks.getJSONObject(0).getString("sha1");
}
}
public Map<String, String> getHeader() {
if (gogs_header == null) {
gogs_header = new HashMap<>();
gogs_header.put("Content-Type", " application/x-www-form-urlencoded");
gogs_header.put("Authorization", "Basic " + new String(Base64.getEncoder().encode((username + ":" + password).getBytes())));
}
return gogs_header;
}
private JSONObject getUser() {
return JSON.parseObject(get("/user"));
}
private List<String> getNameListFromGogs() {
List<String> exist = new ArrayList<>();
JSONArray arr = JSON.parseArray(get("/user/repos"));
for (int i = 0; i < arr.size(); i++) {
JSONObject obj = arr.getJSONObject(i);
exist.add(obj.getString("name"));
}
return exist;
}
public List<RepoInfo> getListFromCoding(String uid) {
Map<String, String> coding_header = new HashMap<>();
coding_header.put("Authorization", "token " + coding_token);
String json = HttpKit.get("https://coding.net/api/projects?page=1&pageSize=1000&type=all", null, coding_header);
JSONArray projs = JSON.parseObject(json).getJSONObject("data").getJSONArray("list");
System.out.println("Coding仓库数量: " + projs.size());
List<RepoInfo> list = new ArrayList<>();
for (int i = 0; i < projs.size(); i++) {
JSONObject proj = projs.getJSONObject(i);
list.add(new RepoInfo(proj.getString("https_url"), proj.getString("name"), username, password, uid, !proj.getBooleanValue("is_public"), proj.getString("description")));
}
return list;
}
public void migrateList(List<RepoInfo> projs) {
long start = System.currentTimeMillis();
ExecutorService executor = Executors.newFixedThreadPool(10);
for (RepoInfo repoInfo : projs) {
executor.execute(() -> migrate(repoInfo));
migrate(repoInfo);
}
executor.shutdown();
try {
boolean loop;
do { //等待所有任务完成
loop = !executor.awaitTermination(2, TimeUnit.SECONDS); //阻塞，直到线程池里所有任务结束
} while (loop);
} catch (InterruptedException ignored) {
}
System.out.println(String.format("迁移完成 耗时 %s 毫秒!", System.currentTimeMillis() - start));
}
public void migrate(RepoInfo info) {
String data = new StringBuilder()
.append("clone_addr=")
.append(eu(info.getUrl()))
.append("&auth_username=")
.append(eu(info.getUser()))
.append("&auth_password=")
.append(eu(info.getPass()))
.append("&uid=")
.append(info.getUid())
.append("&repo_name=")
.append(eu(info.getName()))
.append("&private=")
.append(info.isPrivate() ? "on" : "off")
.append("&mirror=on&description=")
.append(eu(info.getDescription()))
.toString();
try {
get("/repos/" + username + "/" + info.getName());
System.out.println("仓库: " + info.getName() + " 已存在!");
} catch (Exception ex) {
try {
post("/repos/migrate", data);
} catch (Exception ignored) {
while (ignored.getCause() != null) {
ignored = (Exception) ignored.getCause();
}
System.out.println("仓库: " + info.getName() + " 异常: " + ignored.getClass().getName() + ":" + ignored.getMessage());
// 忽略超时以及其他异常
}
System.out.println("仓库: " + info.getName() + " 迁移完成!");
}
}
public String get(String url) {
String result = HttpKit.get(site + "/api/v1" + url, getHeader());
System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + "\n[DEBUG] Result: " + result);
return result;
}
public String delete(String url) {
String result = HttpKit.delete(site + "/api/v1" + url, getHeader());
System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + "\n[DEBUG] Result: " + result);
return result;
}
public String post(String url, String data) {
String result = HttpKit.post(site + "/api/v1" + url, data, getHeader());
System.out.println("[DEBUG] URL: " + site + "/api/v1" + url + " DATA: " + data + "\n[DEBUG] Result: " + result);
return result;
}
public static String eu(String str) {
try {
return URLEncoder.encode(str, "UTF-8");
} catch (UnsupportedEncodingException e) {
return "";
}
}
@Data
@AllArgsConstructor
public static class RepoInfo {
private String url;
private String name;
private String user;
private String pass;
private String uid;
private boolean isPrivate;
private String description;
}
}