Skip to content

Commit

Permalink
[Improve] git clone connect type improvement (#3450)
Browse files Browse the repository at this point in the history 
* [Improve] git clone connect type improvement

* [Improve] minor improve

* [Improve] StringUtils minor improvement

* [Improve] maven build args minor improve

---------

Co-authored-by: benjobs <[email protected]>
  • Loading branch information
@wolfboys
wolfboys and benjobs authored Jan 6, 2024
1 parent 063756b commit b0deb39
Show file tree
Hide file tree
Showing 31 changed files with 181 additions and 243 deletions.
2 changes: 1 addition & 1 deletion streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ insert into `t_flink_effective` values (100000, 100000, 2, 100000, now());
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', '1', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());

-- ----------------------------
-- Records of t_flink_sql
Expand Down
2 changes: 1 addition & 1 deletion streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ insert into "public"."t_flink_effective" values (100000, 100000, 2, 100000, now(
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
insert into "public"."t_flink_project" values (100000, 100000, 'streampark-quickstart', '1', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
insert into "public"."t_flink_project" values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());


-- ----------------------------
Expand Down
1 change: 0 additions & 1 deletion ...mpark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,6 @@ create table `t_flink_project` (
`id` bigint not null auto_increment,
`team_id` bigint not null,
`name` varchar(255) collate utf8mb4_general_ci default null,
`git_credential` tinyint not null,
`url` varchar(255) collate utf8mb4_general_ci default null,
`branches` varchar(64) collate utf8mb4_general_ci default null,
`user_name` varchar(64) collate utf8mb4_general_ci default null,
Expand Down
1 change: 0 additions & 1 deletion ...mpark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,6 @@ create table "public"."t_flink_project" (
"id" int8 not null default nextval('streampark_t_flink_project_id_seq'::regclass),
"team_id" int8,
"name" varchar(255) collate "pg_catalog"."default",
"git_credential" int2,
"url" varchar(255) collate "pg_catalog"."default",
"branches" varchar(64) collate "pg_catalog"."default",
"user_name" varchar(64) collate "pg_catalog"."default",
Expand Down
85 changes: 40 additions & 45 deletions ...mpark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.streampark.common.util.FileUtils;
import org.apache.streampark.common.util.SystemPropertyUtils;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.GitCredentialEnum;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -83,55 +82,51 @@ public static List<String> getBranchList(Project project) throws GitAPIException
}

private static void setCredentials(TransportCommand<?, ?> transportCommand, Project project) {
GitCredentialEnum gitCredentialEnum = GitCredentialEnum.of(project.getGitCredential());
switch (gitCredentialEnum) {
case HTTPS:
if (!StringUtils.isAllBlank(project.getUserName(), project.getPassword())) {
UsernamePasswordCredentialsProvider credentialsProvider =
new UsernamePasswordCredentialsProvider(project.getUserName(), project.getPassword());
transportCommand.setCredentialsProvider(credentialsProvider);
}
break;
case SSH:
transportCommand.setTransportConfigCallback(
transport -> {
SshTransport sshTransport = (SshTransport) transport;
sshTransport.setSshSessionFactory(
new JschConfigSessionFactory() {
@Override
protected void configure(OpenSshConfig.Host hc, Session session) {
session.setConfig("StrictHostKeyChecking", "no");
}
if (project.isHttpRepositoryUrl()) {
if (!StringUtils.isAllEmpty(project.getUserName(), project.getPassword())) {
UsernamePasswordCredentialsProvider credentialsProvider =
new UsernamePasswordCredentialsProvider(project.getUserName(), project.getPassword());
transportCommand.setCredentialsProvider(credentialsProvider);
}
} else if (project.isSshRepositoryUrl()) {
transportCommand.setTransportConfigCallback(
transport -> {
SshTransport sshTransport = (SshTransport) transport;
sshTransport.setSshSessionFactory(
new JschConfigSessionFactory() {
@Override
protected void configure(OpenSshConfig.Host hc, Session session) {
session.setConfig("StrictHostKeyChecking", "no");
}

@Override
protected JSch createDefaultJSch(FS fs) throws JSchException {
JSch jSch = super.createDefaultJSch(fs);
String prvkeyPath = project.getPrvkeyPath();
if (StringUtils.isBlank(prvkeyPath)) {
String userHome = SystemPropertyUtils.getUserHome();
if (userHome != null) {
String rsaPath = userHome.concat("/.ssh/id_rsa");
if (FileUtils.exists(rsaPath)) {
prvkeyPath = rsaPath;
}
@Override
protected JSch createDefaultJSch(FS fs) throws JSchException {
JSch jSch = super.createDefaultJSch(fs);
String prvkeyPath = project.getPrvkeyPath();
if (StringUtils.isBlank(prvkeyPath)) {
String userHome = SystemPropertyUtils.getUserHome();
if (userHome != null) {
String rsaPath = userHome.concat("/.ssh/id_rsa");
if (FileUtils.exists(rsaPath)) {
prvkeyPath = rsaPath;
}
}
if (prvkeyPath == null) {
return jSch;
}
if (StringUtils.isBlank(project.getPassword())) {
jSch.addIdentity(prvkeyPath);
} else {
jSch.addIdentity(prvkeyPath, project.getPassword());
}
}
if (prvkeyPath == null) {
return jSch;
}
});
});
break;
default:
throw new IllegalStateException(
"[StreamPark] git setCredentials: unsupported protocol type");
if (StringUtils.isBlank(project.getPassword())) {
jSch.addIdentity(prvkeyPath);
} else {
jSch.addIdentity(prvkeyPath, project.getPassword());
}
return jSch;
}
});
});
} else {
throw new IllegalStateException(
"[StreamPark] repository URL is invalid, must be ssh or http(s)");
}
}
}
78 changes: 41 additions & 37 deletions ...park-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.streampark.common.conf.CommonConfig;
import org.apache.streampark.common.conf.InternalConfigHolder;
import org.apache.streampark.common.conf.Workspace;
import org.apache.streampark.common.util.CommandUtils;
import org.apache.streampark.common.util.Utils;
import org.apache.streampark.console.base.exception.ApiDetailException;
import org.apache.streampark.console.base.util.GitUtils;
Expand Down Expand Up @@ -68,8 +67,6 @@ public class Project implements Serializable {

private Date lastBuild;

private Integer gitCredential;

@TableField(updateStrategy = FieldStrategy.IGNORED)
private String userName;

Expand Down Expand Up @@ -195,13 +192,21 @@ public String getMavenArgs() {
if (mavenHome == null) {
mavenHome = System.getenv("MAVEN_HOME");
}

boolean useWrapper = true;
if (mavenHome != null) {
mvn = mavenHome + "/bin/" + mvn;
try {
Process process = Runtime.getRuntime().exec(mvn + " --version");
process.waitFor();
Utils.required(process.exitValue() == 0);
useWrapper = false;
} catch (Exception ignored) {
log.warn("try using user-installed maven failed, now use maven-wrapper.");
}
}

try {
CommandUtils.execute(mvn + " --version");
} catch (Exception e) {
if (useWrapper) {
if (windows) {
mvn = WebUtils.getAppHome().concat("/bin/mvnw.cmd");
} else {
Expand All @@ -212,62 +217,53 @@ public String getMavenArgs() {
StringBuilder cmdBuffer = new StringBuilder(mvn).append(" clean package -DskipTests ");

if (StringUtils.isNotBlank(this.buildArgs)) {
String dangerArgs = getDangerArgs(this.buildArgs);
if (dangerArgs == null) {
cmdBuffer.append(this.buildArgs.trim());
} else {
String args = getIllegalArgs(this.buildArgs);
if (args != null) {
throw new IllegalArgumentException(
String.format(
"Invalid maven argument, dangerous args: %s, in your buildArgs: %s",
dangerArgs, this.buildArgs));
"Illegal argument: \"%s\" in maven build parameters: %s", args, this.buildArgs));
}
cmdBuffer.append(this.buildArgs.trim());
}

String setting = InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH());
if (StringUtils.isNotBlank(setting)) {
String dangerArgs = getDangerArgs(setting);
if (dangerArgs == null) {
File file = new File(setting);
if (file.exists() && file.isFile()) {
cmdBuffer.append(" --settings ").append(setting);
} else {
throw new IllegalArgumentException(
String.format("Invalid maven-setting file path, %s no exists or not file", setting));
}
String args = getIllegalArgs(setting);
if (args != null) {
throw new IllegalArgumentException(
String.format("Illegal argument \"%s\" in maven-setting file path: %s", args, setting));
}
File file = new File(setting);
if (file.exists() && file.isFile()) {
cmdBuffer.append(" --settings ").append(setting);
} else {
throw new IllegalArgumentException(
String.format(
"Invalid maven-setting file path, dangerous args: %s, in your maven setting path: %s",
dangerArgs, setting));
"Invalid maven-setting file path \"%s\", the path not exist or is not file",
setting));
}
}
return cmdBuffer.toString();
}

private String getDangerArgs(String param) {
private String getIllegalArgs(String param) {
Pattern pattern = Pattern.compile("(`.*?`)|(\\$\\((.*?)\\))");
Matcher matcher = pattern.matcher(param);
if (matcher.find()) {
String dangerArgs = matcher.group(1);
if (dangerArgs == null) {
dangerArgs = matcher.group(2);
}
return dangerArgs;
return matcher.group(1) == null ? matcher.group(2) : matcher.group(1);
}

String result = null;
Iterator<String> dangerIter = Arrays.asList(";", "|", "&", ">").iterator();
Iterator<String> iterator = Arrays.asList(";", "|", "&", ">").iterator();
String[] argsList = param.split("\\s+");
while (result == null && dangerIter.hasNext()) {
String danger = dangerIter.next();
while (iterator.hasNext()) {
String chr = iterator.next();
for (String arg : argsList) {
if (arg.contains(danger)) {
result = arg;
break;
if (arg.contains(chr)) {
return arg;
}
}
}
return result;
return null;
}

@JsonIgnore
Expand Down Expand Up @@ -298,4 +294,12 @@ public String getLog4CloneStart() {
private String getLogHeader(String header) {
return "---------------------------------[ " + header + " ]---------------------------------\n";
}

public boolean isHttpRepositoryUrl() {
return url != null && (url.trim().startsWith("https://") || url.trim().startsWith("http://"));
}

public boolean isSshRepositoryUrl() {
return url != null && url.trim().startsWith("git@");
}
}
4 changes: 1 addition & 3 deletions ...ice/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildStateEnum;
import org.apache.streampark.console.core.enums.GitCredentialEnum;
import org.apache.streampark.console.core.enums.ReleaseStateEnum;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ProjectService;
Expand Down Expand Up @@ -118,14 +117,13 @@ public boolean update(Project projectParam) {
project.setName(projectParam.getName());
project.setUrl(projectParam.getUrl());
project.setBranches(projectParam.getBranches());
project.setGitCredential(projectParam.getGitCredential());
project.setPrvkeyPath(projectParam.getPrvkeyPath());
project.setUserName(projectParam.getUserName());
project.setPassword(projectParam.getPassword());
project.setPom(projectParam.getPom());
project.setDescription(projectParam.getDescription());
project.setBuildArgs(projectParam.getBuildArgs());
if (GitCredentialEnum.isSSH(project.getGitCredential())) {
if (project.isSshRepositoryUrl()) {
project.setUserName(null);
} else {
project.setPrvkeyPath(null);
Expand Down
5 changes: 1 addition & 4 deletions ...nsole-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.streampark.console.base.util.GitUtils;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildStateEnum;
import org.apache.streampark.console.core.enums.GitCredentialEnum;

import ch.qos.logback.classic.Logger;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -107,9 +106,7 @@ private boolean cloneSourceCode(Project project) {
return true;
} catch (Exception e) {
if (e instanceof InvalidRemoteException) {
GitCredentialEnum gitCredential = GitCredentialEnum.of(project.getGitCredential());
if (gitCredential == GitCredentialEnum.HTTPS) {
project.setGitCredential(GitCredentialEnum.SSH.getValue());
if (project.isHttpRepositoryUrl()) {
String url =
project
.getUrl()
Expand Down
2 changes: 1 addition & 1 deletion streampark-console/streampark-console-service/src/main/resources/db/data-h2.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ insert into `t_flink_effective` values (100000, 100000, 2, 100000, now());
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', '1', 'https://github.com/apache/incubator-streampark-quickstart', 'dev', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'dev', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());

-- ----------------------------
-- Records of t_flink_sql
Expand Down
1 change: 0 additions & 1 deletion streampark-console/streampark-console-service/src/main/resources/db/schema-h2.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,6 @@ create table if not exists `t_flink_project` (
`id` bigint generated by default as identity not null,
`team_id` bigint not null,
`name` varchar(255) default null,
`git_credential` tinyint not null,
`url` varchar(255) default null,
`branches` varchar(64) default null,
`user_name` varchar(64) default null,
Expand Down
1 change: 0 additions & 1 deletion ...mpark-console/streampark-console-service/src/main/resources/mapper/core/ProjectMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
<id column="id" jdbcType="INTEGER" property="id"/>
<result column="team_id" jdbcType="BIGINT" property="teamId"/>
<result column="name" jdbcType="VARCHAR" property="name"/>
<result column="git_credential" jdbcType="INTEGER" property="gitCredential"/>
<result column="url" jdbcType="VARCHAR" property="url"/>
<result column="repository" jdbcType="INTEGER" property="repository"/>
<result column="branches" jdbcType="VARCHAR" property="branches"/>
Expand Down
2 changes: 0 additions & 2 deletions ...-console-service/src/test/java/org/apache/streampark/console/core/entity/ProjectTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.streampark.console.core.entity;

import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum;
import org.apache.streampark.console.core.enums.GitCredentialEnum;

import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -35,7 +34,6 @@ class ProjectTest {
@BeforeEach
void before() {
project.setUrl("https://github.com/apache/incubator-streampark.git");
project.setGitCredential(GitCredentialEnum.HTTPS.getValue());
}

@Disabled("This test case can't be runnable due to external service is not available.")
Expand Down
4 changes: 3 additions & 1 deletion ...rk-console/streampark-console-webapp/src/components/Application/src/AppDarkModeToggle.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,9 @@
height: 18px;
background-color: #fff;
border-radius: 50%;
transition: transform 0.5s, background-color 0.5s;
transition:
transform 0.5s,
background-color 0.5s;
will-change: transform;
}
Expand Down
4 changes: 3 additions & 1 deletion streampark-console/streampark-console-webapp/src/components/ContextMenu/src/ContextMenu.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,9 @@
background-color: @component-background;
border: 1px solid rgb(0 0 0 / 8%);
border-radius: 0.25rem;
box-shadow: 0 2px 2px 0 rgb(0 0 0 / 14%), 0 3px 1px -2px rgb(0 0 0 / 10%),
box-shadow:
0 2px 2px 0 rgb(0 0 0 / 14%),
0 3px 1px -2px rgb(0 0 0 / 10%),
0 1px 5px 0 rgb(0 0 0 / 6%);
background-clip: padding-box;
user-select: none;
Expand Down
2 changes: 1 addition & 1 deletion streampark-console/streampark-console-webapp/src/components/Form/src/BasicForm.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@
});
const getBindValue = computed(
() => ({ ...attrs, ...props, ...unref(getProps) } as Recordable),
() => ({ ...attrs, ...props, ...unref(getProps) }) as Recordable,
);
const getSchema = computed((): FormSchema[] => {
Expand Down
Loading

0 comments on commit b0deb39

Please sign in to comment.