mirror of
https://github.com/go-gitea/gitea
synced 2024-11-01 07:44:25 +00:00
70b7df0e5e
Close #278 Close #24076 ## Solutions: - Use [google/licenseclassifier](https://github.com/google/licenseclassifier/) Test result between [google/licensecheck](https://github.com/google/licensecheck) and [go-license-detector](https://github.com/go-enry/go-license-detector): https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167 Test result between [google/licensecheck](https://github.com/google/licensecheck) and [google/licenseclassifier](https://github.com/google/licenseclassifier/): https://github.com/go-gitea/gitea/pull/24872#issuecomment-1576092178 - Generate License Convert Name List to avoid import license templates with same contents Gitea automatically get latest license data from[ spdx/license-list-data](https://github.com/spdx/license-list-data). But unfortunately, some license templates have same contents. #20915 [click here to see the list](https://github.com/go-gitea/gitea/pull/24872#issuecomment-1584141684) So we will generate a list of these license templates with same contents and create a new file to save the result when using `make generate-license`. (Need to decide the save path) - Save License info into a new table `repo_license` Can easily support searching repo by license in the future. ## Screen shot Single License: ![image](https://github.com/go-gitea/gitea/assets/18380374/41260bd7-0b4c-4038-8592-508706cffa9f) Multiple Licenses: ![image](https://github.com/go-gitea/gitea/assets/18380374/34ce2f73-7e18-446b-9b96-ecc4fb61bd70) Triggers: - [x] Push commit to default branch - [x] Create repo - [x] Mirror repo - [x] When Default Branch is changed, licenses should be updated Todo: - [x] Save Licenses info in to DB when there's a change to license file in the commit - [x] DB Migration - [x] A nominal test? - [x] Select which library to use(https://github.com/go-gitea/gitea/pull/24872#issuecomment-1560361167) - [x] API Support - [x] Add repo license table - ~Select license in settings if there are several licenses(Not recommended)~ - License board(later, not in this PR) ![image](https://github.com/go-gitea/gitea/assets/18380374/2c3c3bf8-bcc2-4c6d-8ce0-81d1a9733878) --------- Co-authored-by: silverwind <me@silverwind.io> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: Denys Konovalov <kontakt@denyskon.de> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: KN4CK3R <admin@oldschoolhack.me> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: 6543 <m.huber@kithara.com> Co-authored-by: a1012112796 <1012112796@qq.com> Co-authored-by: techknowlogick <techknowlogick@gitea.com>
177 lines
4.3 KiB
Go
177 lines
4.3 KiB
Go
// Copyright 2017 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
//go:build ignore
|
|
|
|
package main
|
|
|
|
import (
|
|
"archive/tar"
|
|
"compress/gzip"
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"flag"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"code.gitea.io/gitea/build/license"
|
|
"code.gitea.io/gitea/modules/json"
|
|
"code.gitea.io/gitea/modules/util"
|
|
)
|
|
|
|
func main() {
|
|
var (
|
|
prefix = "gitea-licenses"
|
|
url = "https://api.github.com/repos/spdx/license-list-data/tarball"
|
|
githubApiToken = ""
|
|
githubUsername = ""
|
|
destination = ""
|
|
)
|
|
|
|
flag.StringVar(&destination, "dest", "options/license/", "destination for the licenses")
|
|
flag.StringVar(&githubUsername, "username", "", "github username")
|
|
flag.StringVar(&githubApiToken, "token", "", "github api token")
|
|
flag.Parse()
|
|
|
|
file, err := os.CreateTemp(os.TempDir(), prefix)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create temp file. %s", err)
|
|
}
|
|
|
|
defer util.Remove(file.Name())
|
|
|
|
if err := os.RemoveAll(destination); err != nil {
|
|
log.Fatalf("Cannot clean destination folder: %v", err)
|
|
}
|
|
|
|
if err := os.MkdirAll(destination, 0o755); err != nil {
|
|
log.Fatalf("Cannot create destination: %v", err)
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
log.Fatalf("Failed to download archive. %s", err)
|
|
}
|
|
|
|
if len(githubApiToken) > 0 && len(githubUsername) > 0 {
|
|
req.SetBasicAuth(githubUsername, githubApiToken)
|
|
}
|
|
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
log.Fatalf("Failed to download archive. %s", err)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
if _, err := io.Copy(file, resp.Body); err != nil {
|
|
log.Fatalf("Failed to copy archive to file. %s", err)
|
|
}
|
|
|
|
if _, err := file.Seek(0, 0); err != nil {
|
|
log.Fatalf("Failed to reset seek on archive. %s", err)
|
|
}
|
|
|
|
gz, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
log.Fatalf("Failed to gunzip the archive. %s", err)
|
|
}
|
|
|
|
tr := tar.NewReader(gz)
|
|
aliasesFiles := make(map[string][]string)
|
|
for {
|
|
hdr, err := tr.Next()
|
|
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
|
|
if err != nil {
|
|
log.Fatalf("Failed to iterate archive. %s", err)
|
|
}
|
|
|
|
if !strings.Contains(hdr.Name, "/text/") {
|
|
continue
|
|
}
|
|
|
|
if filepath.Ext(hdr.Name) != ".txt" {
|
|
continue
|
|
}
|
|
|
|
fileBaseName := filepath.Base(hdr.Name)
|
|
licenseName := strings.TrimSuffix(fileBaseName, ".txt")
|
|
|
|
if strings.HasPrefix(fileBaseName, "README") {
|
|
continue
|
|
}
|
|
|
|
if strings.HasPrefix(fileBaseName, "deprecated_") {
|
|
continue
|
|
}
|
|
out, err := os.Create(path.Join(destination, licenseName))
|
|
if err != nil {
|
|
log.Fatalf("Failed to create new file. %s", err)
|
|
}
|
|
|
|
defer out.Close()
|
|
|
|
// some license files have same content, so we need to detect these files and create a convert map into a json file
|
|
// Later we use this convert map to avoid adding same license content with different license name
|
|
h := md5.New()
|
|
// calculate md5 and write file in the same time
|
|
r := io.TeeReader(tr, h)
|
|
if _, err := io.Copy(out, r); err != nil {
|
|
log.Fatalf("Failed to write new file. %s", err)
|
|
} else {
|
|
fmt.Printf("Written %s\n", out.Name())
|
|
|
|
md5 := hex.EncodeToString(h.Sum(nil))
|
|
aliasesFiles[md5] = append(aliasesFiles[md5], licenseName)
|
|
}
|
|
}
|
|
|
|
// generate convert license name map
|
|
licenseAliases := make(map[string]string)
|
|
for _, fileNames := range aliasesFiles {
|
|
if len(fileNames) > 1 {
|
|
licenseName := license.GetLicenseNameFromAliases(fileNames)
|
|
if licenseName == "" {
|
|
// license name should not be empty as expected
|
|
// if it is empty, we need to rewrite the logic of GetLicenseNameFromAliases
|
|
log.Fatalf("GetLicenseNameFromAliases: license name is empty")
|
|
}
|
|
for _, fileName := range fileNames {
|
|
licenseAliases[fileName] = licenseName
|
|
}
|
|
}
|
|
}
|
|
// save convert license name map to file
|
|
b, err := json.Marshal(licenseAliases)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create json bytes. %s", err)
|
|
}
|
|
|
|
licenseAliasesDestination := filepath.Join(destination, "etc", "license-aliases.json")
|
|
if err := os.MkdirAll(filepath.Dir(licenseAliasesDestination), 0o755); err != nil {
|
|
log.Fatalf("Failed to create directory for license aliases json file. %s", err)
|
|
}
|
|
|
|
f, err := os.Create(licenseAliasesDestination)
|
|
if err != nil {
|
|
log.Fatalf("Failed to create license aliases json file. %s", err)
|
|
}
|
|
defer f.Close()
|
|
|
|
if _, err = f.Write(b); err != nil {
|
|
log.Fatalf("Failed to write license aliases json file. %s", err)
|
|
}
|
|
|
|
fmt.Println("Done")
|
|
}
|