f/twins-upstream
1
0
Fork 0
mirror of https://code.rocketnine.space/tslocum/twins.git synced 2024-11-27 11:28:14 +01:00
Code Issues Projects Releases Wiki Activity

Serve Gemini content via HTTPS

Browse source
This commit is contained in:
Trevor Slocum 2020-12-03 11:12:13 -08:00
parent 19b89bfd9e
commit 8d6cb6527e
8 changed files with 372 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

84
CONFIGURATION.md
View file

@ -71,11 +71,19 @@ certbot certonly --config-dir /home/www/certs \
Provide the path to the certificate file at `certs/live/$DOMAIN/fullchain.pem`
and the private key file at `certs/live/$DOMAIN/privkey.pem` to twins.
## DisableHTTPS
Pages are also available via HTTPS on the same port by default.
Set this option to `true` to disable this feature.
Pages are converted automatically by [gmitohtml](https://gitlab.com/tslocum/gmitohtml).
### DisableSize
The size of the response body is included in the media type header by default.
Set this option to `true` to disable this feature. See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md)
for more information.
Set this option to `true` to disable this feature.
See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md) for more information.
### Path
@ -115,42 +123,6 @@ Cache duration (in seconds). Set to `0` to disable caching entirely. This is an
out-of-spec feature. See [PROPOSALS.md](https://gitlab.com/tslocum/twins/blob/master/PROPOSALS.md)
for more information.
##### Hidden
When enabled, hidden files and directories may be accessed. This attribute is
disabled by default.
##### Input
Request text input from user.
##### SensitiveInput
Request sensitive text input from the user. Text will not be shown as it is entered.
##### List
When enabled, directories without an index file will serve a list of their
contents. This attribute is disabled by default.
##### Lang
Specifies content language. This is sent to clients via the MIME type `lang` parameter.
##### Log
Path to log file. Requests are logged in [Apache format](https://httpd.apache.org/docs/2.2/logs.html#combined),
excluding IP address and query.
##### SymLinks
When enabled, symbolic links may be accessed. This attribute is disabled by default.
##### Type
Content type is normally detected automatically. This attribute forces a
specific content type for a path.
##### FastCGI
Forward requests to [FastCGI](https://en.wikipedia.org/wiki/FastCGI) server at
@ -170,6 +142,42 @@ Connect via TCP:
tcp://127.0.0.1:9000
```
##### Hidden
When enabled, hidden files and directories may be accessed. This attribute is
disabled by default.
##### Input
Request text input from user.
##### Lang
Specifies content language. This is sent to clients via the MIME type `lang` parameter.
##### List
When enabled, directories without an index file will serve a list of their
contents. This attribute is disabled by default.
##### Log
Path to log file. Requests are logged in [Apache format](https://httpd.apache.org/docs/2.2/logs.html#combined),
excluding IP address and query.
##### SensitiveInput
Request sensitive text input from the user. Text will not be shown as it is entered.
##### SymLinks
When enabled, symbolic links may be accessed. This attribute is disabled by default.
##### Type
Content type is normally detected automatically. This attribute forces a
specific content type for a path.
## End-of-line indicator
The Gemini protocol requires `\r\n` (CRLF) as the end-of-line indicator. This

4
README.md
View file

@ -18,8 +18,8 @@ This page is also available at [gemini://twins.rocketnine.space](gemini://twins.
- Reverse proxy requests
- TCP
- [FastCGI](https://en.wikipedia.org/wiki/FastCGI)
- Serve system command output
- Redirect to path or URL
- Serve Gemini content via HTTPS
- Pages are converted automatically by [gmitohtml](https://gitlab.com/tslocum/gmitohtml)
- Reload configuration on `SIGHUP`
## Proposals

67
config.go
View file

@ -26,36 +26,36 @@ type pathConfig struct {
Proxy string
Redirect string
// Cache duration
Cache string
// FastCGI server address
FastCGI string
// Serve hidden files and directories
Hidden bool
// Request input
Input string
// Language
Lang string
// List directory
List bool
// Log file
Log string
// Request sensitive input
SensitiveInput string
// Follow symbolic links
SymLinks bool
// Serve hidden files and directories
Hidden bool
// List directory
List bool
// Content type
Type string
// Cache duration
Cache string
// FastCGI server address
FastCGI string
// Language
Lang string
// Log file
Log string
r *regexp.Regexp
cmd []string
cache int64
@ -71,13 +71,10 @@ type hostConfig struct {
type serverConfig struct {
Listen string
Types map[string]string
Hosts map[string]*hostConfig
DisableHTTPS bool
DisableSize bool
SaneEOL bool
hostname string
@ -167,36 +164,36 @@ func readconfig(configPath string) error {
if len(defaultHost.Paths) == 1 {
defaultPath := defaultHost.Paths[0]
for _, serve := range host.Paths {
// Resources
if defaultPath.Root != "" && serve.Root == "" {
serve.Root = defaultPath.Root
}
if defaultPath.Command != "" && serve.Command == "" {
} else if defaultPath.Command != "" && serve.Command == "" {
serve.Command = defaultPath.Command
}
if defaultPath.Proxy != "" && serve.Proxy == "" {
} else if defaultPath.Proxy != "" && serve.Proxy == "" {
serve.Proxy = defaultPath.Proxy
}
if defaultPath.SymLinks {
serve.SymLinks = defaultPath.SymLinks
}
if defaultPath.Hidden {
serve.Hidden = defaultPath.Hidden
}
if defaultPath.List {
serve.List = defaultPath.List
}
// Attributes
if defaultPath.Cache != "" && serve.Cache == "" {
serve.Cache = defaultPath.Cache
}
if defaultPath.FastCGI != "" && serve.FastCGI == "" {
serve.FastCGI = defaultPath.FastCGI
}
if defaultPath.Hidden {
serve.Hidden = defaultPath.Hidden
}
if defaultPath.Lang != "" && serve.Lang == "" {
serve.Lang = defaultPath.Lang
}
if defaultPath.List {
serve.List = defaultPath.List
}
if defaultPath.Log != "" && serve.Log == "" {
serve.Log = defaultPath.Log
}
if defaultPath.SymLinks {
serve.SymLinks = defaultPath.SymLinks
}
}
} else if len(defaultHost.Paths) > 1 {
log.Fatal("only one path may be defined for the default host")

3
go.mod
View file

@ -6,6 +6,7 @@ require (
github.com/h2non/filetype v1.1.0
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107
golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 // indirect
gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c
golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d // indirect
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
)

6
go.sum
View file

@ -12,6 +12,8 @@ github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9
github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107 h1:wfqP/vw5tHVeFQJbnmyXSi7E2ZeshpKhR4kuUR5B7yQ=
github.com/yookoala/gofast v0.4.1-0.20201013050739-975113c54107/go.mod h1:OJU201Q6HCaE1cASckaTbMm3KB6e0cZxK0mgqfwOKvQ=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c h1:hew6E9kPxaR/fipooKTLB+ARkpmI9ShtzYZFdKG7jzQ=
gitlab.com/tslocum/gmitohtml v1.0.3-0.20201203184239-2a1abe8efe7c/go.mod h1:+LFeUUQ6kcjFUR2y3XVr/4i8qLmZOUnM/gwsb9leHMk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@ -34,8 +36,8 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200908211811-12e1bf57a112/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11 h1:gqcmLJzeDSNhSzkyhJ4kxP6CtTimi/5hWFDGp0lFd1w=
golang.org/x/tools v0.0.0-20201117152513-9036a0f9af11/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d h1:OuIGT9zWmMvqaajHPt4H4W1omjiKwkGpBw5ttuErmnw=
golang.org/x/tools v0.0.0-20201203170353-bdde1628ed1d/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

12
serve_fcgi.go
View file

@ -11,23 +11,23 @@ import (
"github.com/yookoala/gofast"
)
type responseWriter struct {
type fakeResponseWriter struct {
io.WriteCloser
header http.Header
}
func newResponseWriter(out io.WriteCloser) *responseWriter {
return &responseWriter{
func newFakeResponseWriter(out io.WriteCloser) *fakeResponseWriter {
return &fakeResponseWriter{
WriteCloser: out,
header: make(http.Header),
}
}
func (w *responseWriter) Header() http.Header {
func (w *fakeResponseWriter) Header() http.Header {
return w.header
}
func (w *responseWriter) WriteHeader(statusCode int) {
func (w *fakeResponseWriter) WriteHeader(statusCode int) {
// Do nothing
}
@ -53,5 +53,5 @@ func serveFastCGI(c net.Conn, connFactory gofast.ConnFactory, u *url.URL, filePa
gofast.NewFileEndpoint(filePath)(gofast.BasicSession),
gofast.SimpleClientFactory(connFactory, 0),
).
ServeHTTP(newResponseWriter(c), r)
ServeHTTP(newFakeResponseWriter(c), r)
}

212
serve_https.go Normal file
View file

@ -0,0 +1,212 @@
package main
import (
"crypto/tls"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"strings"
"gitlab.com/tslocum/gmitohtml/pkg/gmitohtml"
)
var cssBytes = []byte(gmitohtml.StyleCSS)
func serveHTTPS(w http.ResponseWriter, r *http.Request) (int, int64, string) {
if r.URL.Path == "" {
// Redirect to /
u, err := url.Parse(r.URL.String())
if err != nil {
status := http.StatusInternalServerError
http.Error(w, "Failed to parse URL", status)
return status, -1, ""
}
u.Path += "/"
status := http.StatusTemporaryRedirect
http.Redirect(w, r, u.String(), status)
return status, -1, ""
} else if r.URL.Path == "/assets/style.css" {
status := http.StatusOK
w.Header().Set("Content-Type", cssType)
w.WriteHeader(status)
w.Write(cssBytes)
return status, int64(len(cssBytes)), ""
}
pathBytes := []byte(r.URL.Path)
strippedPath := r.URL.Path
if strippedPath[0] == '/' {
strippedPath = strippedPath[1:]
}
if host, ok := config.Hosts[r.URL.Hostname()]; ok {
for _, serve := range host.Paths {
matchedRegexp := serve.r != nil && serve.r.Match(pathBytes)
matchedPrefix := serve.r == nil && strings.HasPrefix(r.URL.Path, serve.Path)
if !matchedRegexp && !matchedPrefix {
continue
}
requireInput := serve.Input != "" || serve.SensitiveInput != ""
if r.URL.RawQuery == "" && requireInput {
if serve.SensitiveInput != "" {
// TODO
}
status := http.StatusInternalServerError
http.Error(w, "Gemini to HTML conversion is not supported for this page", status)
return status, -1, serve.Log
}
if matchedRegexp || matchedPrefix {
if serve.Root == "" || serve.FastCGI != "" {
status := http.StatusInternalServerError
http.Error(w, "Gemini to HTML conversion is not supported for this page", status)
return status, -1, serve.Log
}
var filePath string
if serve.Root != "" {
root := serve.Root
if root[len(root)-1] != '/' {
root += "/"
}
requestSplit := strings.Split(r.URL.Path, "/")
if !serve.SymLinks {
for i := 1; i < len(requestSplit); i++ {
info, err := os.Lstat(path.Join(root, strings.Join(requestSplit[1:i+1], "/")))
if err != nil || info.Mode()&os.ModeSymlink == os.ModeSymlink {
http.NotFound(w, r)
return http.StatusNotFound, -1, serve.Log
}
}
}
filePath = path.Join(root, strings.Join(requestSplit[1:], "/"))
}
fi, err := os.Stat(filePath)
if err != nil {
http.NotFound(w, r)
return http.StatusNotFound, -1, serve.Log
}
mode := fi.Mode()
hasTrailingSlash := len(r.URL.Path) > 0 && r.URL.Path[len(r.URL.Path)-1] == '/'
if mode.IsDir() {
if !hasTrailingSlash {
u, err := url.Parse(r.URL.String())
if err != nil {
status := http.StatusInternalServerError
http.Error(w, "Failed to parse URL", status)
return status, -1, serve.Log
}
u.Path += "/"
status := http.StatusTemporaryRedirect
http.Redirect(w, r, u.String(), status)
return status, -1, serve.Log
}
_, err := os.Stat(path.Join(filePath, "index.gmi"))
if err != nil {
_, err := os.Stat(path.Join(filePath, "index.gemini"))
if err != nil {
if serve.List {
status := http.StatusInternalServerError
http.Error(w, "HTTPS dir lost not yet implemented", status)
return status, -1, serve.Log
}
http.NotFound(w, r)
return http.StatusNotFound, -1, serve.Log
}
filePath = path.Join(filePath, "index.gemini")
} else {
filePath = path.Join(filePath, "index.gmi")
}
} else if hasTrailingSlash && len(r.URL.Path) > 1 {
u, err := url.Parse(r.URL.String())
if err != nil {
status := http.StatusInternalServerError
http.Error(w, "Failed to parse URL", status)
return status, -1, serve.Log
}
u.Path = u.Path[:len(u.Path)-1]
status := http.StatusTemporaryRedirect
http.Redirect(w, r, u.String(), status)
return status, -1, serve.Log
}
data, err := ioutil.ReadFile(filePath)
if err != nil {
status := http.StatusInternalServerError
http.Error(w, err.Error(), status)
return status, -1, serve.Log
}
result := gmitohtml.Convert([]byte(data), r.URL.String())
status := http.StatusOK
w.Header().Set("Content-Type", htmlType)
w.WriteHeader(status)
w.Write(result)
return status, int64(len(result)), serve.Log
}
}
}
http.NotFound(w, r)
return http.StatusNotFound, -1, ""
}
type responseWriter struct {
statusCode int
header http.Header
conn *tls.Conn
wroteHeader bool
}
func newResponseWriter(conn *tls.Conn) *responseWriter {
return &responseWriter{
header: http.Header{},
conn: conn,
}
}
func (w *responseWriter) Header() http.Header {
return w.header
}
func (w *responseWriter) Write(b []byte) (int, error) {
if !w.wroteHeader {
w.wroteHeader = true
w.WriteHeader(http.StatusOK)
}
return w.conn.Write(b)
}
func (w *responseWriter) WriteHeader(statusCode int) {
if w.wroteHeader {
return
}
w.statusCode = statusCode
statusText := http.StatusText(statusCode)
if statusText == "" {
statusText = "Unknown"
}
w.conn.Write([]byte(fmt.Sprintf("HTTP/1.1 %d %s\r\n", statusCode, statusText)))
w.header.Write(w.conn)
w.conn.Write([]byte("\r\n"))
}

83
server.go
View file

@ -6,8 +6,10 @@ import (
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"log"
"net"
"net/http"
"net/url"
"os"
"path"
@ -27,6 +29,7 @@ const (
plainType = "text/plain; charset=utf-8"
geminiType = "text/gemini; charset=utf-8"
htmlType = "text/html; charset=utf-8"
cssType = "text/css; charset=utf-8"
logTimeFormat = "2006-01-02 15:04:05"
)
@ -303,13 +306,14 @@ func handleConn(c *tls.Conn) {
var logPath string
status := 0
size := int64(-1)
protocol := "Gemini"
defer func() {
if quiet && logPath == "" {
return
}
entry := logEntry(request, status, size, time.Since(t))
entry := logEntry(protocol, request, status, size, time.Since(t))
if !quiet {
log.Println(string(entry))
@ -336,22 +340,34 @@ func handleConn(c *tls.Conn) {
f.Write([]byte("\n"))
}()
defer c.Close()
c.SetReadDeadline(time.Now().Add(readTimeout))
defer c.Close()
var requestData string
scanner := bufio.NewScanner(c)
if !config.SaneEOL {
scanner.Split(scanCRLF)
var dataBuf []byte
var buf = make([]byte, 1)
var readCR bool
if config.SaneEOL {
readCR = true
}
if scanner.Scan() {
requestData = scanner.Text()
}
if err := scanner.Err(); err != nil {
status = writeStatus(c, statusBadRequest)
for {
n, err := c.Read(buf)
if err == io.EOF {
break
} else if err != nil || n != 1 {
return
}
if buf[0] == '\r' {
readCR = true
continue
} else if readCR && buf[0] == '\n' {
break
}
dataBuf = append(dataBuf, buf[0])
}
requestData := string(dataBuf)
state := c.ConnectionState()
certs := state.PeerCertificates
var clientCertKeys [][]byte
@ -363,6 +379,36 @@ func handleConn(c *tls.Conn) {
clientCertKeys = append(clientCertKeys, pubKey)
}
if strings.HasPrefix(requestData, "GET ") {
w := newResponseWriter(c)
defer w.WriteHeader(http.StatusOK)
if config.DisableHTTPS {
status = statusProxyRequestRefused
http.Error(w, "Error: Proxy request refused", http.StatusBadRequest)
return
}
r, err := http.ReadRequest(bufio.NewReader(io.MultiReader(strings.NewReader(requestData+"\r\n"), c)))
if err != nil {
status = http.StatusInternalServerError
http.Error(w, err.Error(), status)
return
}
if r.Proto == "" {
protocol = "HTTP"
} else {
protocol = r.Proto
}
r.URL.Scheme = "https"
r.URL.Host = strings.ToLower(r.Host)
request = r.URL
status, size, logPath = serveHTTPS(w, r)
return
}
if len(requestData) > urlMaxLength || !utf8.ValidString(requestData) {
status = writeStatus(c, statusBadRequest)
return
@ -374,6 +420,7 @@ func handleConn(c *tls.Conn) {
status = writeStatus(c, statusBadRequest)
return
}
request.Host = strings.ToLower(request.Host)
requestHostname := request.Hostname()
if requestHostname == "" || strings.ContainsRune(requestHostname, ' ') {
@ -389,22 +436,16 @@ func handleConn(c *tls.Conn) {
}
}
if request.Scheme == "" {
request.Scheme = "gemini"
}
if request.Scheme != "gemini" || (requestPort > 0 && requestPort != config.port) {
validScheme := request.Scheme == "gemini" || (!config.DisableHTTPS && request.Scheme == "https")
if !validScheme || (requestPort > 0 && requestPort != config.port) {
status = writeStatus(c, statusProxyRequestRefused)
return
}
if request.Scheme == "gemini" {
request.Host = strings.ToLower(request.Host)
}
status, size, logPath = handleRequest(c, request, requestData)
}
func logEntry(request *url.URL, status int, size int64, elapsed time.Duration) []byte {
func logEntry(protocol string, request *url.URL, status int, size int64, elapsed time.Duration) []byte {
hostFormatted := "-"
pathFormatted := "-"
sizeFormatted := "-"
@ -424,7 +465,7 @@ func logEntry(request *url.URL, status int, size int64, elapsed time.Duration) [
if size >= 0 {
sizeFormatted = strconv.FormatInt(size, 10)
}
return []byte(fmt.Sprintf(`%s - - - [%s] "GET %s Gemini" %d %s %.4f`, hostFormatted, time.Now().Format(logTimeFormat), pathFormatted, status, sizeFormatted, elapsed.Seconds()))
return []byte(fmt.Sprintf(`%s - - - [%s] "GET %s %s" %d %s %.4f`, hostFormatted, time.Now().Format(logTimeFormat), pathFormatted, protocol, status, sizeFormatted, elapsed.Seconds()))
}
func handleListener(l net.Listener) {