diff options
Diffstat (limited to 'src/ext_depends/arsd/cgi.d')
| -rw-r--r-- | src/ext_depends/arsd/cgi.d | 1185 |
1 files changed, 916 insertions, 269 deletions
diff --git a/src/ext_depends/arsd/cgi.d b/src/ext_depends/arsd/cgi.d index 9189052..844a411 100644 --- a/src/ext_depends/arsd/cgi.d +++ b/src/ext_depends/arsd/cgi.d @@ -54,7 +54,7 @@ void main() { */ /++ - Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications. + Provides a uniform server-side API for CGI, FastCGI, SCGI, and HTTP web applications. Offers both lower- and higher- level api options among other common (optional) things like websocket and event source serving support, session management, and job scheduling. --- import arsd.cgi; @@ -74,6 +74,30 @@ void main() { mixin GenericMain!hello; --- + Or: + --- + import arsd.cgi; + + class MyApi : WebObject { + @UrlName("") + string hello(string name = null) { + if(name is null) + return "Hello, world!"; + else + return "Hello, " ~ name; + } + } + mixin DispatcherMain!( + "/".serveApi!MyApi + ); + --- + + $(NOTE + Please note that using the higher-level api will add a dependency on arsd.dom and arsd.jsvar to your application. + If you use `dmd -i` or `ldc2 -i` to build, it will just work, but with dub, you will have do `dub add arsd-official:jsvar` + and `dub add arsd-official:dom` yourself. + ) + Test on console (works in any interface mode): $(CONSOLE $ ./cgi_hello GET / name=whatever @@ -85,10 +109,12 @@ void main() { # now you can go to http://localhost:8080/?name=whatever ) - Please note: the default port for http is 8085 and for cgi is 4000. I recommend you set your own by the command line argument in a startup script instead of relying on any hard coded defaults. It is possible though to hard code your own with [RequestServer]. + Please note: the default port for http is 8085 and for scgi is 4000. I recommend you set your own by the command line argument in a startup script instead of relying on any hard coded defaults. It is possible though to code your own with [RequestServer], however. - Compile_versions: + Build_Configurations: + + cgi.d tries to be flexible to meet your needs. It is possible to configure it both at runtime (by writing your own `main` function and constructing a [RequestServer] object) or at compile time using the `version` switch to the compiler or a dub `subConfiguration`. If you are using `dub`, use: @@ -105,11 +131,11 @@ void main() { to change versions. The possible options for `VALUE_HERE` are: $(LIST - * `embedded_httpd` for the embedded httpd version (built-in web server). This is the default. - * `cgi` for traditional cgi binaries. - * `fastcgi` for FastCGI builds. - * `scgi` for SCGI builds. - * `stdio_http` for speaking raw http over stdin and stdout. See [RequestServer.serveSingleHttpConnectionOnStdio] for more information. + * `embedded_httpd` for the embedded httpd version (built-in web server). This is the default for dub builds. You can run the program then connect directly to it from your browser. + * `cgi` for traditional cgi binaries. These are run by an outside web server as-needed to handle requests. + * `fastcgi` for FastCGI builds. FastCGI is managed from an outside helper, there's one built into Microsoft IIS, Apache httpd, and Lighttpd, and a generic program you can use with nginx called `spawn-fcgi`. If you don't already know how to use it, I suggest you use one of the other modes. + * `scgi` for SCGI builds. SCGI is a simplified form of FastCGI, where you run the server as an application service which is proxied by your outside webserver. + * `stdio_http` for speaking raw http over stdin and stdout. This is made for systemd services. See [RequestServer.serveSingleHttpConnectionOnStdio] for more information. ) With dmd, use: @@ -127,7 +153,8 @@ void main() { - A FastCGI executable will be generated. * - `-version=scgi` - A SCGI (SimpleCGI) executable will be generated. - + * - `-version=embedded_httpd_hybrid` + - A HTTP server that uses a combination of processes, threads, and fibers to better handle large numbers of idle connections. Recommended if you are going to serve websockets in a non-local application. * - `-version=embedded_httpd_threads` - The embedded HTTP server will use a single process with a thread pool. (use instead of plain `embedded_httpd` if you want this specific implementation) * - `-version=embedded_httpd_processes` @@ -141,7 +168,7 @@ void main() { + (can be used together with others) * - `-version=cgi_with_websocket` - - The CGI class has websocket server support. + - The CGI class has websocket server support. (This is on by default now.) * - `-version=with_openssl` - not currently used @@ -151,7 +178,7 @@ void main() { - The session will be provided in a separate process, provided by cgi.d. ) - Compile_and_run: + For example, For CGI, `dmd yourfile.d cgi.d` then put the executable in your cgi-bin directory. @@ -161,77 +188,24 @@ void main() { For an embedded HTTP server, run `dmd yourfile.d cgi.d -version=embedded_httpd` and run the generated program. It listens on port 8085 by default. You can change this on the command line with the --port option when running your program. - You can also simulate a request by passing parameters on the command line, like: + Simulating_requests: + + If you are using one of the [GenericMain] or [DispatcherMain] mixins, or main with your own call to [RequestServer.trySimulatedRequest], you can simulate requests from your command-ine shell. Call the program like this: $(CONSOLE ./yourprogram GET / name=adr ) - And it will print the result to stdout. + And it will print the result to stdout instead of running a server, regardless of build more.. CGI_Setup_tips: - On Apache, you may do `SetHandler cgi-script` in your `.htaccess` file. + On Apache, you may do `SetHandler cgi-script` in your `.htaccess` file to set a particular file to be run through the cgi program. Note that all "subdirectories" of it also run the program; if you configure `/foo` to be a cgi script, then going to `/foo/bar` will call your cgi handler function with `cgi.pathInfo == "/bar"`. - Integration_tips: + Overview_Of_Basic_Concepts: - cgi.d works well with dom.d for generating html. You may also use web.d for other utilities and automatic api wrapping. + cgi.d offers both lower-level handler apis as well as higher-level auto-dispatcher apis. For a lower-level handler function, you'll probably want to review the following functions: - dom.d usage: - - --- - import arsd.cgi; - import arsd.dom; - - void hello_dom(Cgi cgi) { - auto document = new Document(); - - static import std.file; - // parse the file in strict mode, requiring it to be well-formed UTF-8 XHTML - // (You'll appreciate this if you've ever had to deal with a missing </div> - // or something in a php or erb template before that would randomly mess up - // the output in your browser. Just check it and throw an exception early!) - // - // You could also hard-code a template or load one at compile time with an - // import expression, but you might appreciate making it a regular file - // because that means it can be more easily edited by the frontend team and - // they can see their changes without needing to recompile the program. - // - // Note on CTFE: if you do choose to load a static file at compile time, - // you *can* parse it in CTFE using enum, which will cause it to throw at - // compile time, which is kinda cool too. Be careful in modifying that document, - // though, as it will be a static instance. You might want to clone on on demand, - // or perhaps modify it lazily as you print it out. (Try element.tree, it returns - // a range of elements which you could send through std.algorithm functions. But - // since my selector implementation doesn't work on that level yet, you'll find that - // harder to use. Of course, you could make a static list of matching elements and - // then use a simple e is e2 predicate... :) ) - document.parseUtf8(std.file.read("your_template.html"), true, true); - - // fill in data using DOM functions, so placing it is in the hands of HTML - // and it will be properly encoded as text too. - // - // Plain html templates can't run server side logic, but I think that's a - // good thing - it keeps them simple. You may choose to extend the html, - // but I think it is best to try to stick to standard elements and fill them - // in with requested data with IDs or class names. A further benefit of - // this is the designer can also highlight data based on sources in the CSS. - // - // However, all of dom.d is available, so you can format your data however - // you like. You can do partial templates with innerHTML too, or perhaps better, - // injecting cloned nodes from a partial document. - // - // There's a lot of possibilities. - document["#name"].innerText = cgi.request("name", "default name"); - - // send the document to the browser. The second argument to `cgi.write` - // indicates that this is all the data at once, enabling a few small - // optimizations. - cgi.write(document.toString(), true); - } - --- - - Concepts: Input: [Cgi.get], [Cgi.post], [Cgi.request], [Cgi.files], [Cgi.cookies], [Cgi.pathInfo], [Cgi.requestMethod], and HTTP headers ([Cgi.headers], [Cgi.userAgent], [Cgi.referrer], [Cgi.accept], [Cgi.authorization], [Cgi.lastEventId]) @@ -245,12 +219,175 @@ void main() { Other Information: [Cgi.remoteAddress], [Cgi.https], [Cgi.port], [Cgi.scriptName], [Cgi.requestUri], [Cgi.getCurrentCompleteUri], [Cgi.onRequestBodyDataReceived] - Overriding behavior: [Cgi.handleIncomingDataChunk], [Cgi.prepareForIncomingDataChunks], [Cgi.cleanUpPostDataState] + Websockets: [Websocket], [websocketRequested], [acceptWebsocket]. For websockets, use the `embedded_httpd_hybrid` build mode for best results, because it is optimized for handling large numbers of idle connections compared to the other build modes. + + Overriding behavior for special cases streaming input data: see the virtual functions [Cgi.handleIncomingDataChunk], [Cgi.prepareForIncomingDataChunks], [Cgi.cleanUpPostDataState] + + A basic program using the lower-level api might look like: + + --- + import arsd.cgi; + + // you write a request handler which always takes a Cgi object + void handler(Cgi cgi) { + /+ + when the user goes to your site, suppose you are being hosted at http://example.com/yourapp + + If the user goes to http://example.com/yourapp/test?name=value + then the url will be parsed out into the following pieces: + + cgi.pathInfo == "/test". This is everything after yourapp's name. (If you are doing an embedded http server, your app's name is blank, so pathInfo will be the whole path of the url.) + + cgi.scriptName == "yourapp". With an embedded http server, this will be blank. + + cgi.host == "example.com" + + cgi.https == false + + cgi.queryString == "name=value" (there's also cgi.search, which will be "?name=value", including the ?) + + The query string is further parsed into the `get` and `getArray` members, so: + + cgi.get == ["name": "value"], meaning you can do `cgi.get["name"] == "value"` + + And + + cgi.getArray == ["name": ["value"]]. + + Why is there both `get` and `getArray`? The standard allows names to be repeated. This can be very useful, + it is how http forms naturally pass multiple items like a set of checkboxes. So `getArray` is the complete data + if you need it. But since so often you only care about one value, the `get` member provides more convenient access. + + We can use these members to process the request and build link urls. Other info from the request are in other members, we'll look at them later. + +/ + switch(cgi.pathInfo) { + // the home page will be a small html form that can set a cookie. + case "/": + cgi.write(`<!DOCTYPE html> + <html> + <body> + <form method="POST" action="set-cookie"> + <label>Your name: <input type="text" name="name" /></label> + <input type="submit" value="Submit" /> + </form> + </body> + </html> + `, true); // the , true tells it that this is the one, complete response i want to send, allowing some optimizations. + break; + // POSTing to this will set a cookie with our submitted name + case "/set-cookie": + // HTTP has a number of request methods (also called "verbs") to tell + // what you should do with the given resource. + // The most common are GET and POST, the ones used in html forms. + // You can check which one was used with the `cgi.requestMethod` property. + if(cgi.requestMethod == Cgi.RequestMethod.POST) { + + // headers like redirections need to be set before we call `write` + cgi.setResponseLocation("read-cookie"); + + // just like how url params go into cgi.get/getArray, form data submitted in a POST + // body go to cgi.post/postArray. Please note that a POST request can also have get + // params in addition to post params. + // + // There's also a convenience function `cgi.request("name")` which checks post first, + // then get if it isn't found there, and then returns a default value if it is in neither. + if("name" in cgi.post) { + // we can set cookies with a method too + // again, cookies need to be set before calling `cgi.write`, since they + // are a kind of header. + cgi.setCookie("name" , cgi.post["name"]); + } + + // the user will probably never see this, since the response location + // is an automatic redirect, but it is still best to say something anyway + cgi.write("Redirecting you to see the cookie...", true); + } else { + // you can write out response codes and headers + // as well as response bodies + // + // But always check the cgi docs before using the generic + // `header` method - if there is a specific method for your + // header, use it before resorting to the generic one to avoid + // a header value from being sent twice. + cgi.setResponseLocation("405 Method Not Allowed"); + // there is no special accept member, so you can use the generic header function + cgi.header("Accept: POST"); + // but content type does have a method, so prefer to use it: + cgi.setResponseContentType("text/plain"); + + // all the headers are buffered, and will be sent upon the first body + // write. you can actually modify some of them before sending if need be. + cgi.write("You must use the POST http verb on this resource.", true); + } + break; + // and GETting this will read the cookie back out + case "/read-cookie": + // I did NOT pass `,true` here because this is writing a partial response. + // It is possible to stream data to the user in chunks by writing partial + // responses the calling `cgi.flush();` to send the partial response immediately. + // normally, you'd only send partial chunks if you have to - it is better to build + // a response as a whole and send it as a whole whenever possible - but here I want + // to demo that you can. + cgi.write("Hello, "); + if("name" in cgi.cookies) { + import arsd.dom; // dom.d provides a lot of helpers for html + // since the cookie is set, we need to write it out properly to + // avoid cross-site scripting attacks. + // + // Getting this stuff right automatically is a benefit of using the higher + // level apis, but this demo is to show the fundamental building blocks, so + // we're responsible to take care of it. + cgi.write(htmlEntitiesEncode(cgi.cookies["name"])); + } else { + cgi.write("friend"); + } + + // note that I never called cgi.setResponseContentType, since the default is text/html. + // it doesn't hurt to do it explicitly though, just remember to do it before any cgi.write + // calls. + break; + default: + // no path matched + cgi.setResponseStatus("404 Not Found"); + cgi.write("Resource not found.", true); + } + } + + // and this adds the boilerplate to set up a server according to the + // compile version configuration and call your handler as requests come in + mixin GenericMain!handler; // the `handler` here is the name of your function + --- + + Even if you plan to always use the higher-level apis, I still recommend you at least familiarize yourself with the lower level functions, since they provide the lightest weight, most flexible options to get down to business if you ever need them. + + In the lower-level api, the [Cgi] object represents your HTTP transaction. It has functions to describe the request and for you to send your response. It leaves the details of how you o it up to you. The general guideline though is to avoid depending any variables outside your handler function, since there's no guarantee they will survive to another handler. You can use global vars as a lazy initialized cache, but you should always be ready in case it is empty. (One exception: if you use `-version=embedded_httpd_threads -version=cgi_no_fork`, then you can rely on it more, but you should still really write things assuming your function won't have anything survive beyond its return for max scalability and compatibility.) + + A basic program using the higher-level apis might look like: - Installing: Apache, IIS, CGI, FastCGI, SCGI, embedded HTTPD (not recommended for production use) + --- + /+ + import arsd.cgi; + + struct LoginData { + string currentUser; + } + + class AppClass : WebObject { + string foo() {} + } + + mixin DispatcherMain!( + "/assets/.serveStaticFileDirectory("assets/", true), // serve the files in the assets subdirectory + "/".serveApi!AppClass, + "/thing/".serveRestObject, + ); + +/ + --- Guide_for_PHP_users: - If you are coming from PHP, here's a quick guide to help you get started: + (Please note: I wrote this section in 2008. A lot of PHP hosts still ran 4.x back then, so it was common to avoid using classes - introduced in php 5 - to maintain compatibility! If you're coming from php more recently, this may not be relevant anymore, but still might help you.) + + If you are coming from old-style PHP, here's a quick guide to help you get started: $(SIDE_BY_SIDE $(COLUMN @@ -326,27 +463,113 @@ void main() { See_Also: - You may also want to see [arsd.dom], [arsd.web], and [arsd.html] for more code for making - web applications. + You may also want to see [arsd.dom], [arsd.webtemplate], and maybe some functions from my old [arsd.html] for more code for making + web applications. dom and webtemplate are used by the higher-level api here in cgi.d. For working with json, try [arsd.jsvar]. [arsd.database], [arsd.mysql], [arsd.postgres], [arsd.mssql], and [arsd.sqlite] can help in accessing databases. - If you are looking to access a web application via HTTP, try [std.net.curl], [arsd.curl], or [arsd.http2]. + If you are looking to access a web application via HTTP, try [arsd.http2]. Copyright: - cgi.d copyright 2008-2021, Adam D. Ruppe. Provided under the Boost Software License. + cgi.d copyright 2008-2023, Adam D. Ruppe. Provided under the Boost Software License. Yes, this file is old, and yes, it is still actively maintained and used. + + History: + An import of `arsd.core` was added on March 21, 2023 (dub v11.0). Prior to this, the module's default configuration was completely stand-alone. You must now include the `core.d` file in your builds with `cgi.d`. + + This change is primarily to integrate the event loops across the library, allowing you to more easily use cgi.d along with my other libraries like simpledisplay and http2.d. Previously, you'd have to run separate helper threads. Now, they can all automatically work together. +/ module arsd.cgi; +static import arsd.core; +version(Posix) +import arsd.core : makeNonBlocking; + +// FIXME: Nullable!T can be a checkbox that enables/disables the T on the automatic form +// and a SumType!(T, R) can be a radio box to pick between T and R to disclose the extra boxes on the automatic form + +/++ + This micro-example uses the [dispatcher] api to act as a simple http file server, serving files found in the current directory and its children. ++/ +version(Demo) +unittest { + import arsd.cgi; + + mixin DispatcherMain!( + "/".serveStaticFileDirectory(null, true) + ); +} + +/++ + Same as the previous example, but written out long-form without the use of [DispatcherMain] nor [GenericMain]. ++/ +version(Demo) +unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + cgi.dispatcher!( + "/".serveStaticFileDirectory(null, true) + ); + } + + // mixin GenericMain!requestHandler would add this function: + void main(string[] args) { + // this is all the content of [cgiMainImpl] which you can also call + + // cgi.d embeds a few add on functions like real time event forwarders + // and session servers it can run in other processes. this spawns them, if needed. + if(tryAddonServers(args)) + return; + + // cgi.d allows you to easily simulate http requests from the command line, + // without actually starting a server. this function will do that. + if(trySimulatedRequest!(requestHandler, Cgi)(args)) + return; + + RequestServer server; + // you can change the default port here if you like + // server.listeningPort = 9000; + + // then call this to let the command line args override your default + server.configureFromCommandLine(args); + + // here is where you could print out the listeningPort to the user if you wanted + + // and serve the request(s) according to the compile configuration + server.serve!(requestHandler)(); + + // or you could explicitly choose a serve mode like this: + // server.serveEmbeddedHttp!requestHandler(); + } +} + +/++ + cgi.d has built-in testing helpers too. These will provide mock requests and mock sessions that + otherwise run through the rest of the internal mechanisms to call your functions without actually + spinning up a server. ++/ version(Demo) unittest { + import arsd.cgi; + + void requestHandler(Cgi cgi) { + + } + + // D doesn't let me embed a unittest inside an example unittest + // so this is a function, but you can do it however in your real program + /* unittest */ void runTests() { + auto tester = new CgiTester(&requestHandler); + auto response = tester.GET("/"); + assert(response.code == 200); + } } static import std.file; @@ -360,9 +583,7 @@ version(Posix) { } else version(minimal) { } else { - version(GNU) { - // GDC doesn't support static foreach so I had to cheat on it :( - } else version(FreeBSD) { + version(FreeBSD) { // I never implemented the fancy stuff there either } else { version=with_breaking_cgi_features; @@ -381,6 +602,7 @@ version(Windows) { } } +// FIXME: can use the arsd.core function now but it is trivial anyway tbh void cloexec(int fd) { version(Posix) { import core.sys.posix.fcntl; @@ -402,6 +624,11 @@ version(embedded_httpd_hybrid) { version=cgi_use_fiber; } +version(cgi_use_fork) + enum cgi_use_fork_default = true; +else + enum cgi_use_fork_default = false; + // the servers must know about the connections to talk to them; the interfaces are vital version(with_addon_servers) version=with_addon_servers_connections; @@ -952,7 +1179,7 @@ class Cgi { { import core.runtime; auto sfn = getenv("SCRIPT_FILENAME"); - scriptFileName = sfn.length ? sfn : Runtime.args[0]; + scriptFileName = sfn.length ? sfn : (Runtime.args.length ? Runtime.args[0] : null); } bool iis = false; @@ -1031,7 +1258,8 @@ class Cgi { // to be slow if they did that. The spec says it is always there though. // And it has worked reliably for me all year in the live environment, // but some servers might be different. - auto contentLength = to!size_t(getenv("CONTENT_LENGTH")); + auto cls = getenv("CONTENT_LENGTH"); + auto contentLength = to!size_t(cls.length ? cls : "0"); immutable originalContentLength = contentLength; if(contentLength) { @@ -1745,7 +1973,7 @@ class Cgi { { import core.runtime; - scriptFileName = Runtime.args[0]; + scriptFileName = Runtime.args.length ? Runtime.args[0] : null; } @@ -1860,6 +2088,8 @@ class Cgi { // FIXME: if size is > max content length it should // also fail at this point. _rawDataOutput(cast(ubyte[]) "HTTP/1.1 100 Continue\r\n\r\n"); + + // FIXME: let the user write out 103 early hints too } } // else @@ -2196,6 +2426,17 @@ class Cgi { customHeaders ~= h; } + /++ + I named the original function `header` after PHP, but this pattern more fits + the rest of the Cgi object. + + Either name are allowed. + + History: + Alias added June 17, 2022. + +/ + alias setResponseHeader = header; + private string[] customHeaders; private bool websocketMode; @@ -2398,7 +2639,7 @@ class Cgi { return; // don't double close if(!outputtedResponseData) - write("", false, false); + write("", true, false); // writing auto buffered data if(requestMethod != RequestMethod.HEAD && autoBuffer) { @@ -2523,6 +2764,23 @@ class Cgi { version(preserveData) // note: this can eat lots of memory; don't use unless you're sure you need it. immutable(ubyte)[] originalPostData; + /++ + This holds the posted body data if it has not been parsed into [post] and [postArray]. + + It is intended to be used for JSON and XML request content types, but also may be used + for other content types your application can handle. But it will NOT be populated + for content types application/x-www-form-urlencoded or multipart/form-data, since those are + parsed into the post and postArray members. + + Remember that anything beyond your `maxContentLength` param when setting up [GenericMain], etc., + will be discarded to the client with an error. This helps keep this array from being exploded in size + and consuming all your server's memory (though it may still be possible to eat excess ram from a concurrent + client in certain build modes.) + + History: + Added January 5, 2021 + Documented February 21, 2023 (dub v11.0) + +/ public immutable string postBody; alias postJson = postBody; // old name @@ -2893,7 +3151,10 @@ struct Uri { host = authority; } else { host = authority[0 .. idx2]; - port = to!int(authority[idx2 + 1 .. $]); + if(idx2 + 1 < authority.length) + port = to!int(authority[idx2 + 1 .. $]); + else + port = 0; } } @@ -2969,6 +3230,8 @@ struct Uri { /// Browsers use a function like this to figure out links in html. Uri basedOn(in Uri baseUrl) const { Uri n = this; // copies + if(n.scheme == "data") + return n; // n.uriInvalidated = true; // make sure we regenerate... // userinfo is not inherited... is this wrong? @@ -3278,7 +3541,11 @@ string toHexUpper(long num) { // the generic mixins -/// Use this instead of writing your own main +/++ + Use this instead of writing your own main + + It ultimately calls [cgiMainImpl] which creates a [RequestServer] for you. ++/ mixin template GenericMain(alias fun, long maxContentLength = defaultMaxContentLength) { mixin CustomCgiMain!(Cgi, fun, maxContentLength); } @@ -3292,6 +3559,23 @@ mixin template GenericMain(alias fun, long maxContentLength = defaultMaxContentL Added July 9, 2021 +/ mixin template DispatcherMain(Presenter, DispatcherArgs...) { + /// forwards to [CustomCgiDispatcherMain] with default args + mixin CustomCgiDispatcherMain!(Cgi, defaultMaxContentLength, Presenter, DispatcherArgs); +} + +/// ditto +mixin template DispatcherMain(DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { + class GenericPresenter : WebPresenter!GenericPresenter {} + mixin DispatcherMain!(GenericPresenter, DispatcherArgs); +} + +/++ + Allows for a generic [DispatcherMain] with custom arguments. Note you can use [defaultMaxContentLength] as the second argument if you like. + + History: + Added May 13, 2023 (dub v11.0) ++/ +mixin template CustomCgiDispatcherMain(CustomCgi, size_t maxContentLength, Presenter, DispatcherArgs...) { /++ Handler to the generated presenter you can use from your objects, etc. +/ @@ -3326,12 +3610,14 @@ mixin template DispatcherMain(Presenter, DispatcherArgs...) { presenter.renderBasicError(cgi, 404); } } - mixin GenericMain!handler; + mixin CustomCgiMain!(CustomCgi, handler, maxContentLength); } -mixin template DispatcherMain(DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { +/// ditto +mixin template CustomCgiDispatcherMain(CustomCgi, size_t maxContentLength, DispatcherArgs...) if(!is(DispatcherArgs[0] : WebPresenter!T, T)) { class GenericPresenter : WebPresenter!GenericPresenter {} - mixin DispatcherMain!(GenericPresenter, DispatcherArgs); + mixin CustomCgiDispatcherMain!(CustomCgi, maxContentLength, GenericPresenter, DispatcherArgs); + } private string simpleHtmlEncode(string s) { @@ -3478,6 +3764,28 @@ struct RequestServer { /// ushort listeningPort = defaultListeningPort(); + /++ + Uses a fork() call, if available, to provide additional crash resiliency and possibly improved performance. On the + other hand, if you fork, you must not assume any memory is shared between requests (you shouldn't be anyway though! But + if you have to, you probably want to set this to false and use an explicit threaded server with [serveEmbeddedHttp]) and + [stop] may not work as well. + + History: + Added August 12, 2022 (dub v10.9). Previously, this was only configurable through the `-version=cgi_no_fork` + argument to dmd. That version still defines the value of `cgi_use_fork_default`, used to initialize this, for + compatibility. + +/ + bool useFork = cgi_use_fork_default; + + /++ + Determines the number of worker threads to spawn per process, for server modes that use worker threads. 0 will use a + default based on the number of cpus modified by the server mode. + + History: + Added August 12, 2022 (dub v10.9) + +/ + int numberOfThreads = 0; + /// this(string defaultHost, ushort defaultPort) { this.listeningHost = defaultHost; @@ -3509,11 +3817,11 @@ struct RequestServer { foundHost = false; } if(foundUid) { - privDropUserId = to!int(arg); + privilegesDropToUid = to!uid_t(arg); foundUid = false; } if(foundGid) { - privDropGroupId = to!int(arg); + privilegesDropToGid = to!gid_t(arg); foundGid = false; } if(arg == "--listening-host" || arg == "-h" || arg == "/listening-host") @@ -3527,7 +3835,37 @@ struct RequestServer { } } - // FIXME: the privDropUserId/group id need to be set in here instead of global + version(Windows) { + private alias uid_t = int; + private alias gid_t = int; + } + + /// user (uid) to drop privileges to + /// 0 … do nothing + uid_t privilegesDropToUid = 0; + /// group (gid) to drop privileges to + /// 0 … do nothing + gid_t privilegesDropToGid = 0; + + private void dropPrivileges() { + version(Posix) { + import core.sys.posix.unistd; + + if (privilegesDropToGid != 0 && setgid(privilegesDropToGid) != 0) + throw new Exception("Dropping privileges via setgid() failed."); + + if (privilegesDropToUid != 0 && setuid(privilegesDropToUid) != 0) + throw new Exception("Dropping privileges via setuid() failed."); + } + else { + // FIXME: Windows? + //pragma(msg, "Dropping privileges is not implemented for this platform"); + } + + // done, set zero + privilegesDropToGid = 0; + privilegesDropToUid = 0; + } /++ Serves a single HTTP request on this thread, with an embedded server, then stops. Designed for cases like embedded oauth responders @@ -3556,7 +3894,7 @@ struct RequestServer { bool tcp; void delegate() cleanup; - auto socket = startListening(listeningHost, listeningPort, tcp, cleanup, 1); + auto socket = startListening(listeningHost, listeningPort, tcp, cleanup, 1, &dropPrivileges); auto connection = socket.accept(); doThreadHttpConnectionGuts!(CustomCgi, fun, true)(connection); @@ -3603,6 +3941,7 @@ struct RequestServer { If you want the forking worker process server, you do need to compile with the embedded_httpd_processes config though. +/ void serveEmbeddedHttp(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(ThisFor!fun _this) { + globalStopFlag = false; static if(__traits(isStaticFunction, fun)) alias funToUse = fun; else @@ -3611,7 +3950,7 @@ struct RequestServer { __traits(child, _this, fun)(cgi); else static assert(0, "Not implemented in your compiler version!"); } - auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, funToUse)); + auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadHttpConnection!(CustomCgi, funToUse), null, useFork, numberOfThreads); manager.listen(); } @@ -3619,14 +3958,15 @@ struct RequestServer { Runs the embedded SCGI server specifically, regardless of which build configuration you have. +/ void serveScgi(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)() { - auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength)); + globalStopFlag = false; + auto manager = new ListeningConnectionManager(listeningHost, listeningPort, &doThreadScgiConnection!(CustomCgi, fun, maxContentLength), null, useFork, numberOfThreads); manager.listen(); } /++ Serves a single "connection", but the connection is spoken on stdin and stdout instead of on a socket. - Intended for cases like working from systemd, like discussed here: https://forum.dlang.org/post/avmkfdiitirnrenzljwc@forum.dlang.org + Intended for cases like working from systemd, like discussed here: [https://forum.dlang.org/post/avmkfdiitirnrenzljwc@forum.dlang.org] History: Added May 29, 2021 @@ -3636,28 +3976,60 @@ struct RequestServer { } /++ - Stops serving after the current requests. + The [stop] function sets a flag that request handlers can (and should) check periodically. If a handler doesn't + respond to this flag, the library will force the issue. This determines when and how the issue will be forced. + +/ + enum ForceStop { + /++ + Stops accepting new requests, but lets ones already in the queue start and complete before exiting. + +/ + afterQueuedRequestsComplete, + /++ + Finishes requests already started their handlers, but drops any others in the queue. Streaming handlers + should cooperate and exit gracefully, but if they don't, it will continue waiting for them. + +/ + afterCurrentRequestsComplete, + /++ + Partial response writes will throw an exception, cancelling any streaming response, but complete + writes will continue to process. Request handlers that respect the stop token will also gracefully cancel. + +/ + cancelStreamingRequestsEarly, + /++ + All writes will throw. + +/ + cancelAllRequestsEarly, + /++ + Use OS facilities to forcibly kill running threads. The server process will be in an undefined state after this call (if this call ever returns). + +/ + forciblyTerminate, + } + + version(embedded_httpd_processes) {} else + /++ + Stops serving after the current requests are completed. Bugs: - Not implemented on version=embedded_httpd_processes, version=fastcgi, or on any operating system aside from Linux at this time. - Try SIGINT there perhaps. + Not implemented on version=embedded_httpd_processes, version=fastcgi on any system, or embedded_httpd on Windows (it does work on embedded_httpd_hybrid + on Windows however). Only partially implemented on non-Linux posix systems. + + You might also try SIGINT perhaps. - A Windows implementation is planned but not sure about the others. Maybe a posix pipe can be used on other OSes. I do not intend - to implement this for the processes config. + The stopPriority is not yet fully implemented. +/ - version(embedded_httpd_processes) {} else - static void stop() { + static void stop(ForceStop stopPriority = ForceStop.afterCurrentRequestsComplete) { globalStopFlag = true; - version(Posix) - if(cancelfd > 0) { - ulong a = 1; - core.sys.posix.unistd.write(cancelfd, &a, a.sizeof); + version(Posix) { + if(cancelfd > 0) { + ulong a = 1; + core.sys.posix.unistd.write(cancelfd, &a, a.sizeof); + } } - version(Windows) - if(iocp) { - foreach(i; 0 .. 16) // FIXME - PostQueuedCompletionStatus(iocp, 0, cast(ULONG_PTR) null, null); + version(Windows) { + if(iocp) { + foreach(i; 0 .. 16) // FIXME + PostQueuedCompletionStatus(iocp, 0, cast(ULONG_PTR) null, null); + } } } } @@ -3679,28 +4051,6 @@ else private __gshared bool globalStopFlag = false; -private int privDropUserId; -private int privDropGroupId; - -// Added Jan 11, 2021 -private void dropPrivs() { - version(Posix) { - import core.sys.posix.unistd; - - auto userId = privDropUserId; - auto groupId = privDropGroupId; - - if((userId != 0 || groupId != 0) && getuid() == 0) { - if(groupId) - setgid(groupId); - if(userId) - setuid(userId); - } - - } - // FIXME: Windows? -} - version(embedded_httpd_processes) void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer params) { import core.sys.posix.unistd; @@ -3744,7 +4094,7 @@ void serveEmbeddedHttpdProcesses(alias fun, CustomCgi = Cgi)(RequestServer param close(sock); throw new Exception("listen"); } - dropPrivs(); + params.dropPrivileges(); } version(embedded_httpd_processes_accept_after_fork) {} else { @@ -4082,6 +4432,8 @@ string defaultListeningHost() { /++ This is the function [GenericMain] calls. View its source for some simple boilerplate you can copy/paste and modify, or you can call it yourself from your `main`. + Please note that this may spawn other helper processes that will call `main` again. It does this currently for the timer server and event source server (and the quasi-deprecated web socket server). + Params: fun = Your request handler CustomCgi = a subclass of Cgi, if you wise to customize it further @@ -4089,7 +4441,7 @@ string defaultListeningHost() { args = command-line arguments History: - Documented Sept 26, 2020. + Documented Sept 26, 2020. +/ void cgiMainImpl(alias fun, CustomCgi = Cgi, long maxContentLength = defaultMaxContentLength)(string[] args) if(is(CustomCgi : Cgi)) { if(tryAddonServers(args)) @@ -4294,7 +4646,7 @@ extern(Windows) private { alias LPWSAOVERLAPPED = LPOVERLAPPED; /+ - alias LPFN_ACCEPTEX = + alias LPFN_ACCEPTEX = BOOL function( SOCKET sListenSocket, @@ -4437,7 +4789,7 @@ private class PseudoblockingOverlappedSocket : Socket { WSABUF[1] buffer; OVERLAPPED overlapped; - override ptrdiff_t send(const(void)[] buf, SocketFlags flags) @trusted { + override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) @trusted { overlapped = overlapped.init; buffer[0].len = cast(DWORD) buf.length; buffer[0].buf = cast(CHAR*) buf.ptr; @@ -4452,7 +4804,7 @@ private class PseudoblockingOverlappedSocket : Socket { Fiber.yield(); return lastAnswer; } - override ptrdiff_t receive(void[] buf, SocketFlags flags) @trusted { + override ptrdiff_t receive(scope void[] buf, SocketFlags flags) @trusted { overlapped = overlapped.init; buffer[0].len = cast(DWORD) buf.length; buffer[0].buf = cast(CHAR*) buf.ptr; @@ -4473,16 +4825,16 @@ private class PseudoblockingOverlappedSocket : Socket { } // I might go back and implement these for udp things. - override ptrdiff_t receiveFrom(void[] buf, SocketFlags flags, ref Address from) @trusted { + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags, ref Address from) @trusted { assert(0); } - override ptrdiff_t receiveFrom(void[] buf, SocketFlags flags) @trusted { + override ptrdiff_t receiveFrom(scope void[] buf, SocketFlags flags) @trusted { assert(0); } - override ptrdiff_t sendTo(const(void)[] buf, SocketFlags flags, Address to) @trusted { + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags, Address to) @trusted { assert(0); } - override ptrdiff_t sendTo(const(void)[] buf, SocketFlags flags) @trusted { + override ptrdiff_t sendTo(scope const(void)[] buf, SocketFlags flags) @trusted { assert(0); } @@ -5108,20 +5460,20 @@ private class FakeSocketForStdin : Socket { private bool closed; - override ptrdiff_t receive(void[] buffer, std.socket.SocketFlags) @trusted { + override ptrdiff_t receive(scope void[] buffer, std.socket.SocketFlags) @trusted { if(closed) throw new Exception("Closed"); return stdin.rawRead(buffer).length; } - override ptrdiff_t send(const void[] buffer, std.socket.SocketFlags) @trusted { + override ptrdiff_t send(const scope void[] buffer, std.socket.SocketFlags) @trusted { if(closed) throw new Exception("Closed"); stdout.rawWrite(buffer); return buffer.length; } - override void close() @trusted { + override void close() @trusted scope { (cast(void delegate() @nogc nothrow) &realClose)(); } @@ -5129,7 +5481,7 @@ private class FakeSocketForStdin : Socket { // FIXME } - override void setOption(SocketOptionLevel, SocketOption, void[]) {} + override void setOption(SocketOptionLevel, SocketOption, scope void[]) {} override void setOption(SocketOptionLevel, SocketOption, Duration) {} override @property @trusted Address remoteAddress() { return null; } @@ -5152,9 +5504,13 @@ import core.atomic; /** To use this thing: + --- void handler(Socket s) { do something... } - auto manager = new ListeningConnectionManager("127.0.0.1", 80, &handler); + auto manager = new ListeningConnectionManager("127.0.0.1", 80, &handler, &delegateThatDropsPrivileges); manager.listen(); + --- + + The 4th parameter is optional. I suggest you use BufferedInputRange(connection) to handle the input. As a packet comes in, you will get control. You can just continue; though to fetch more. @@ -5175,7 +5531,8 @@ class ListeningConnectionManager { fd_set read_fds; FD_ZERO(&read_fds); FD_SET(listener.handle, &read_fds); - FD_SET(cancelfd, &read_fds); + if(cancelfd != -1) + FD_SET(cancelfd, &read_fds); auto max = listener.handle > cancelfd ? listener.handle : cancelfd; auto ret = select(max + 1, &read_fds, null, null, null); if(ret == -1) { @@ -5186,7 +5543,7 @@ class ListeningConnectionManager { throw new Exception("wtf select"); } - if(FD_ISSET(cancelfd, &read_fds)) { + if(cancelfd != -1 && FD_ISSET(cancelfd, &read_fds)) { return null; } @@ -5194,8 +5551,36 @@ class ListeningConnectionManager { return listener.accept(); return null; - } else - return listener.accept(); // FIXME: check the cancel flag! + } else { + + Socket socket = listener; + + auto check = new SocketSet(); + + keep_looping: + check.reset(); + check.add(socket); + + // just to check the stop flag on a kinda busy loop. i hate this FIXME + auto got = Socket.select(check, null, null, 3.seconds); + if(got > 0) + return listener.accept(); + if(globalStopFlag) + return null; + else + goto keep_looping; + } + } + + int defaultNumberOfThreads() { + import std.parallelism; + version(cgi_use_fiber) { + return totalCPUs * 1 + 1; + } else { + // I times 4 here because there's a good chance some will be blocked on i/o. + return totalCPUs * 4; + } + } void listen() { @@ -5229,11 +5614,12 @@ class ListeningConnectionManager { } } } else { - import std.parallelism; - version(cgi_use_fork) { - //asm { int 3; } - fork(); + if(useFork) { + version(linux) { + //asm { int 3; } + fork(); + } } version(cgi_use_fiber) { @@ -5242,7 +5628,7 @@ class ListeningConnectionManager { listener.accept(); } - WorkerThread[] threads = new WorkerThread[](totalCPUs * 1 + 1); + WorkerThread[] threads = new WorkerThread[](numberOfThreads); foreach(i, ref thread; threads) { thread = new WorkerThread(this, handler, cast(int) i); thread.start(); @@ -5269,8 +5655,7 @@ class ListeningConnectionManager { } else { semaphore = new Semaphore(); - // I times 4 here because there's a good chance some will be blocked on i/o. - ConnectionThread[] threads = new ConnectionThread[](totalCPUs * 4); + ConnectionThread[] threads = new ConnectionThread[](numberOfThreads); foreach(i, ref thread; threads) { thread = new ConnectionThread(this, handler, cast(int) i); thread.start(); @@ -5354,17 +5739,20 @@ class ListeningConnectionManager { private void dg_handler(Socket s) { fhandler(s); } - this(string host, ushort port, void function(Socket) handler) { + this(string host, ushort port, void function(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { fhandler = handler; - this(host, port, &dg_handler); + this(host, port, &dg_handler, dropPrivs, useFork, numberOfThreads); } - this(string host, ushort port, void delegate(Socket) handler) { + this(string host, ushort port, void delegate(Socket) handler, void delegate() dropPrivs = null, bool useFork = cgi_use_fork_default, int numberOfThreads = 0) { this.handler = handler; + this.useFork = useFork; + this.numberOfThreads = numberOfThreads ? numberOfThreads : defaultNumberOfThreads(); - listener = startListening(host, port, tcp, cleanup, 128); + listener = startListening(host, port, tcp, cleanup, 128, dropPrivs); - version(cgi_use_fiber) version(cgi_use_fork) + version(cgi_use_fiber) + if(useFork) listener.blocking = false; // this is the UI control thread and thus gets more priority @@ -5373,9 +5761,12 @@ class ListeningConnectionManager { Socket listener; void delegate(Socket) handler; + + immutable bool useFork; + int numberOfThreads; } -Socket startListening(string host, ushort port, ref bool tcp, ref void delegate() cleanup, int backQueue) { +Socket startListening(string host, ushort port, ref bool tcp, ref void delegate() cleanup, int backQueue, void delegate() dropPrivs) { Socket listener; if(host.startsWith("unix:")) { version(Posix) { @@ -5423,7 +5814,8 @@ Socket startListening(string host, ushort port, ref bool tcp, ref void delegate( listener.listen(backQueue); - dropPrivs(); + if (dropPrivs !is null) // can be null, backwards compatibility + dropPrivs(); return listener; } @@ -6573,8 +6965,8 @@ version(Windows) } version(Posix) { + import core.sys.posix.unistd; version(CRuntime_Musl) {} else { - import core.sys.posix.unistd; private extern(C) int posix_spawn(pid_t*, const char*, void*, void*, const char**, const char**); } } @@ -6758,18 +7150,6 @@ void runSessionServer()() { runAddonServer("/tmp/arsd_session_server", new BasicDataServerImplementation()); } -version(Posix) -private void makeNonBlocking(int fd) { - import core.sys.posix.fcntl; - auto flags = fcntl(fd, F_GETFL, 0); - if(flags == -1) - throw new Exception("fcntl get"); - flags |= O_NONBLOCK; - auto s = fcntl(fd, F_SETFL, flags); - if(s == -1) - throw new Exception("fcntl set"); -} - import core.stdc.errno; struct IoOp { @@ -6905,7 +7285,7 @@ interface EventIoServer { } // the sink should buffer it -private void serialize(T)(scope void delegate(ubyte[]) sink, T t) { +private void serialize(T)(scope void delegate(scope ubyte[]) sink, T t) { static if(is(T == struct)) { foreach(member; __traits(allMembers, T)) serialize(sink, __traits(getMember, t, member)); @@ -6984,7 +7364,7 @@ unittest { }, 56674); ubyte[1000] buffer; int bufferPoint; - void add(ubyte[] b) { + void add(scope ubyte[] b) { buffer[bufferPoint .. bufferPoint + b.length] = b[]; bufferPoint += b.length; } @@ -7052,7 +7432,7 @@ mixin template ImplementRpcClientInterface(T, string serverPath, string cmdArg) // derivedMembers on an interface seems to give exactly what I want: the virtual functions we need to implement. so I am just going to use it directly without more filtering. static foreach(idx, member; __traits(derivedMembers, T)) { - static if(__traits(isVirtualFunction, __traits(getMember, T, member))) + static if(__traits(isVirtualMethod, __traits(getMember, T, member))) mixin( q{ std.traits.ReturnType!(__traits(getMember, T, member)) } ~ member ~ q{(std.traits.Parameters!(__traits(getMember, T, member)) params) @@ -7087,9 +7467,9 @@ mixin template ImplementRpcClientInterface(T, string serverPath, string cmdArg) int dataLocation; ubyte[] grab(int sz) { - auto d = got[dataLocation .. dataLocation + sz]; + auto dataLocation1 = dataLocation; dataLocation += sz; - return d; + return got[dataLocation1 .. dataLocation]; } typeof(return) retu; @@ -7145,7 +7525,7 @@ void dispatchRpcServer(Interface, Class)(Class this_, ubyte[] data, int fd) if(i sw: switch(calledIdx) { foreach(idx, memberName; __traits(derivedMembers, Interface)) - static if(__traits(isVirtualFunction, __traits(getMember, Interface, memberName))) { + static if(__traits(isVirtualMethod, __traits(getMember, Interface, memberName))) { case idx: assert(calledFunction == __traits(getMember, Interface, memberName).mangleof); @@ -7955,7 +8335,7 @@ final class EventSourceServerImplementation : EventSourceServer, EventIoServer { int typeLength; char[32] typeBuffer = 0; int messageLength; - char[2048] messageBuffer = 0; + char[2048 * 4] messageBuffer = 0; // this is an arbitrary limit, it needs to fit comfortably in stack (including in a fiber) and be a single send on the kernel side cuz of the impl... i think this is ok for a unix socket. int _lifetime; char[] message() return { @@ -8228,7 +8608,7 @@ void runAddonServer(EIS)(string localListenerName, EIS eis) if(is(EIS : EventIoS cloexec(ns); makeNonBlocking(ns); - auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096, &eis.handleLocalConnectionData); + auto niop = allocateIoOp(ns, IoOp.ReadSocketHandle, 4096 * 4, &eis.handleLocalConnectionData); niop.closeHandler = &eis.handleLocalConnectionClose; niop.completeHandler = &eis.handleLocalConnectionComplete; scope(failure) freeIoOp(niop); @@ -8732,6 +9112,9 @@ auto callFromCgi(alias method, T)(T dg, Cgi cgi) { } return false; + } else static if(is(T == enum)) { + *what = to!T(value); + return true; } else static if(isSomeString!T || isIntegral!T || isFloatingPoint!T) { *what = to!T(value); return true; @@ -9156,8 +9539,10 @@ css"; Element htmlContainer() { auto document = new Document(q"html <!DOCTYPE html> -<html> +<html class="no-script"> <head> + <script>document.documentElement.classList.remove("no-script");</script> + <style>.no-script requires-script { display: none; }</style> <title>D Application</title> <link rel="stylesheet" href="style.css" /> </head> @@ -9190,8 +9575,33 @@ html", true, true); } void presentSuccessfulReturn(T, Meta)(Cgi cgi, T ret, Meta meta, string format) { - // FIXME? format? - (cast(CRTP) this).presentSuccessfulReturnAsHtml(cgi, ret, meta); + switch(format) { + case "html": + (cast(CRTP) this).presentSuccessfulReturnAsHtml(cgi, ret, meta); + break; + case "json": + import arsd.jsvar; + static if(is(typeof(ret) == MultipleResponses!Types, Types...)) { + var json; + foreach(index, type; Types) { + if(ret.contains == index) + json = ret.payload[index]; + } + } else { + var json = ret; + } + var envelope = json; // var.emptyObject; + /* + envelope.success = true; + envelope.result = json; + envelope.error = null; + */ + cgi.setResponseContentType("application/json"); + cgi.write(envelope.toJson(), true); + break; + default: + cgi.setResponseStatus("406 Not Acceptable"); // not exactly but sort of. + } } /// typeof(null) (which is also used to represent functions returning `void`) do nothing @@ -9231,9 +9641,14 @@ html", true, true); assert(0); } - /// An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort. + /++ + An instance of the [arsd.dom.FileResource] interface has its own content type; assume it is a download of some sort if the filename member is non-null of the FileResource interface. + +/ void presentSuccessfulReturn(T : FileResource, Meta)(Cgi cgi, T ret, Meta meta, string format) { cgi.setCache(true); // not necessarily true but meh + if(auto fn = ret.filename()) { + cgi.header("Content-Disposition: attachment; filename="~fn~";"); + } cgi.setResponseContentType(ret.contentType); cgi.write(ret.getData(), true); } @@ -9246,6 +9661,21 @@ html", true, true); } /++ + + History: + Added January 23, 2023 (dub v11.0) + +/ + void presentExceptionalReturn(Meta)(Cgi cgi, Throwable t, Meta meta, string format) { + switch(format) { + case "html": + presentExceptionAsHtml(cgi, t, meta); + break; + default: + } + } + + + /++ If you override this, you will need to cast the exception type `t` dynamically, but can then use the template arguments here to refer back to the function. @@ -9253,14 +9683,29 @@ html", true, true); method on the live object. You could, in theory, change arguments and retry, but I provide that information mostly with the expectation that you will use them to make useful forms or richer error messages for the user. + + History: + BREAKING CHANGE on January 23, 2023 (v11.0 ): it previously took an `alias func` and `T dg` to call the function again. + I removed this in favor of a `Meta` param. + + Before: `void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg)` + + After: `void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta)` + + If you used the func for something, move that something into your `methodMeta` template. + + What is the benefit of this change? Somewhat smaller executables and faster builds thanks to more reused functions, together with + enabling an easier implementation of [presentExceptionalReturn]. +/ - void presentExceptionAsHtml(alias func, T)(Cgi cgi, Throwable t, T dg) { + void presentExceptionAsHtml(Meta)(Cgi cgi, Throwable t, Meta meta) { Form af; + /+ foreach(attr; __traits(getAttributes, func)) { static if(__traits(isSame, attr, AutomaticForm)) { af = createAutomaticFormForFunction!(func)(dg); } } + +/ presentExceptionAsHtmlImpl(cgi, t, af); } @@ -9363,6 +9808,21 @@ html", true, true); auto i = lbl.addChild("input", name); i.attrs.name = name; i.attrs.type = "file"; + } else static if(is(T == enum)) { + Element lbl; + if(displayName !is null) { + lbl = div.addChild("label"); + lbl.addChild("span", displayName, "label-text"); + lbl.appendText(" "); + } else { + lbl = div; + } + auto i = lbl.addChild("select", name); + i.attrs.name = name; + + foreach(memberName; __traits(allMembers, T)) + i.addChild("option", memberName); + } else static if(is(T == struct)) { if(displayName !is null) div.addChild("span", displayName, "label-text"); @@ -9411,18 +9871,6 @@ html", true, true); i.attrs.type = "checkbox"; i.attrs.value = "true"; i.attrs.name = name; - } else static if(is(T == Cgi.UploadedFile)) { - Element lbl; - if(displayName !is null) { - lbl = div.addChild("label"); - lbl.addChild("span", displayName, "label-text"); - lbl.appendText(" "); - } else { - lbl = div; - } - auto i = lbl.addChild("input", name); - i.attrs.name = name; - i.attrs.type = "file"; } else static if(is(T == K[], K)) { auto templ = div.addChild("template"); templ.appendChild(elementFor!(K)(null, name, null /* uda??*/)); @@ -9686,30 +10134,34 @@ struct MultipleResponses(T...) { --- auto valueToTest = your_test_function(); - valueToTest.visit!( - (Redirection) { assert(0); }, // got a redirection instead of a string, fail the test + valueToTest.visit( + (Redirection r) { assert(0); }, // got a redirection instead of a string, fail the test (string s) { assert(s == "test"); } // right value, go ahead and test it. ); --- + + History: + Was horribly broken until June 16, 2022. Ironically, I wrote it for tests but never actually tested it. + It tried to use alias lambdas before, but runtime delegates work much better so I changed it. +/ - void visit(Handlers...)() { - template findHandler(type, HandlersToCheck...) { + void visit(Handlers...)(Handlers handlers) { + template findHandler(type, int count, HandlersToCheck...) { static if(HandlersToCheck.length == 0) - alias findHandler = void; + enum findHandler = -1; else { - static if(is(typeof(HandlersToCheck[0](type.init)))) - alias findHandler = handler; + static if(is(typeof(HandlersToCheck[0].init(type.init)))) + enum findHandler = count; else - alias findHandler = findHandler!(type, HandlersToCheck[1 .. $]); + enum findHandler = findHandler!(type, count + 1, HandlersToCheck[1 .. $]); } } foreach(index, type; T) { - alias handler = findHandler!(type, Handlers); - static if(is(handler == void)) + enum handlerIndex = findHandler!(type, 0, Handlers); + static if(handlerIndex == -1) static assert(0, "Type " ~ type.stringof ~ " was not handled by visitor"); else { - if(index == contains) - handler(payload[index]); + if(index == this.contains) + handlers[handlerIndex](this.payload[index]); } } } @@ -9795,7 +10247,12 @@ private string nextPieceFromSlash(ref string remainingUrl) { return ident; } +/++ + UDA used to indicate to the [dispatcher] that a trailing slash should always be added to or removed from the url. It will do it as a redirect header as-needed. ++/ enum AddTrailingSlash; +/// ditto +enum RemoveTrailingSlash; private auto serveApiInternal(T)(string urlPrefix) { @@ -9852,7 +10309,7 @@ private auto serveApiInternal(T)(string urlPrefix) { switch(cgi.request("format", "html")) { case "html": static void dummy() {} - presenter.presentExceptionAsHtml!(dummy)(cgi, t, &dummy); + presenter.presentExceptionAsHtml(cgi, t, null); return true; case "json": var envelope = var.emptyObject; @@ -9976,6 +10433,12 @@ private auto serveApiInternal(T)(string urlPrefix) { cgi.setResponseLocation(cgi.pathInfo ~ "/"); return true; } + } else static if(is(attr == RemoveTrailingSlash)) { + if(remainingUrl !is null) { + cgi.setResponseLocation(cgi.pathInfo[0 .. lastIndexOf(cgi.pathInfo, "/")]); + return true; + } + } else static if(__traits(isSame, AutomaticForm, attr)) { automaticForm = true; } @@ -10054,47 +10517,25 @@ private auto serveApiInternal(T)(string urlPrefix) { if(callFunction) +/ - if(automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET) { + auto format = cgi.request("format", defaultFormat!overload()); + auto wantsFormFormat = format.startsWith("form-"); + + if(wantsFormFormat || (automaticForm && cgi.requestMethod == Cgi.RequestMethod.GET)) { // Should I still show the form on a json thing? idk... auto ret = presenter.createAutomaticFormForFunction!((__traits(getOverloads, obj, methodName)[idx]))(&(__traits(getOverloads, obj, methodName)[idx])); - presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), "html"); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), wantsFormFormat ? format["form_".length .. $] : "html"); return true; } - switch(cgi.request("format", defaultFormat!overload())) { - case "html": - // a void return (or typeof(null) lol) means you, the user, is doing it yourself. Gives full control. - try { - - auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi); - presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), "html"); - } catch(Throwable t) { - presenter.presentExceptionAsHtml!(__traits(getOverloads, obj, methodName)[idx])(cgi, t, &(__traits(getOverloads, obj, methodName)[idx])); - } - return true; - case "json": - auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi); - static if(is(typeof(ret) == MultipleResponses!Types, Types...)) { - var json; - foreach(index, type; Types) { - if(ret.contains == index) - json = ret.payload[index]; - } - } else { - var json = ret; - } - var envelope = json; // var.emptyObject; - /* - envelope.success = true; - envelope.result = json; - envelope.error = null; - */ - cgi.setResponseContentType("application/json"); - cgi.write(envelope.toJson(), true); - return true; - default: - cgi.setResponseStatus("406 Not Acceptable"); // not exactly but sort of. - return true; + + try { + // a void return (or typeof(null) lol) means you, the user, is doing it yourself. Gives full control. + auto ret = callFromCgi!(__traits(getOverloads, obj, methodName)[idx])(&(__traits(getOverloads, obj, methodName)[idx]), cgi); + presenter.presentSuccessfulReturn(cgi, ret, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); + } catch(Throwable t) { + // presenter.presentExceptionAsHtml!(__traits(getOverloads, obj, methodName)[idx])(cgi, t, &(__traits(getOverloads, obj, methodName)[idx])); + presenter.presentExceptionalReturn(cgi, t, presenter.methodMeta!(__traits(getOverloads, obj, methodName)[idx]), format); } + return true; //}} //cgi.header("Accept: POST"); // FIXME list the real thing @@ -10743,14 +11184,12 @@ bool restObjectServeHandler(T, Presenter)(Cgi cgi, Presenter presenter, string u // FIXME: OPTIONS, HEAD } catch(Throwable t) { - presenter.presentExceptionAsHtml!(DUMMY)(cgi, t, null); + presenter.presentExceptionAsHtml(cgi, t); } return true; } -struct DUMMY {} - /+ struct SetOfFields(T) { private void[0][string] storage; @@ -10770,6 +11209,33 @@ enum hideonindex; +/ /++ + Returns true if I recommend gzipping content of this type. You might + want to call it from your Presenter classes before calling cgi.write. + + --- + cgi.setResponseContentType(yourContentType); + cgi.gzipResponse = gzipRecommendedForContentType(yourContentType); + cgi.write(yourData, true); + --- + + This is used internally by [serveStaticFile], [serveStaticFileDirectory], [serveStaticData], and maybe others I forgot to update this doc about. + + + The implementation considers text content to be recommended to gzip. This may change, but it seems reasonable enough for now. + + History: + Added January 28, 2023 (dub v11.0) ++/ +bool gzipRecommendedForContentType(string contentType) { + if(contentType.startsWith("text/")) + return true; + if(contentType.startsWith("application/javascript")) + return true; + + return false; +} + +/++ Serves a static file. To be used with [dispatcher]. See_Also: [serveApi], [serveRestObject], [dispatcher], [serveRedirect] @@ -10779,7 +11245,7 @@ auto serveStaticFile(string urlPrefix, string filename = null, string contentTyp // man 2 sendfile assert(urlPrefix[0] == '/'); if(filename is null) - filename = urlPrefix[1 .. $]; + filename = decodeComponent(urlPrefix[1 .. $]); // FIXME is this actually correct? if(contentType is null) { contentType = contentTypeFromFileExtension(filename); } @@ -10790,9 +11256,10 @@ auto serveStaticFile(string urlPrefix, string filename = null, string contentTyp } static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { - if(details.contentType.indexOf("image/") == 0) + if(details.contentType.indexOf("image/") == 0 || details.contentType.indexOf("audio/") == 0) cgi.setCache(true); cgi.setResponseContentType(details.contentType); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); cgi.write(std.file.read(details.filename), true); return true; } @@ -10828,6 +11295,8 @@ auto serveStaticData(string urlPrefix, immutable(void)[] data, string contentTyp string contentTypeFromFileExtension(string filename) { if(filename.endsWith(".png")) return "image/png"; + if(filename.endsWith(".apng")) + return "image/apng"; if(filename.endsWith(".svg")) return "image/svg+xml"; if(filename.endsWith(".jpg")) @@ -10842,28 +11311,56 @@ string contentTypeFromFileExtension(string filename) { return "application/wasm"; if(filename.endsWith(".mp3")) return "audio/mpeg"; + if(filename.endsWith(".pdf")) + return "application/pdf"; return null; } /// This serves a directory full of static files, figuring out the content-types from file extensions. /// It does not let you to descend into subdirectories (or ascend out of it, of course) -auto serveStaticFileDirectory(string urlPrefix, string directory = null) { +auto serveStaticFileDirectory(string urlPrefix, string directory = null, bool recursive = false) { assert(urlPrefix[0] == '/'); assert(urlPrefix[$-1] == '/'); static struct DispatcherDetails { string directory; + bool recursive; } if(directory is null) directory = urlPrefix[1 .. $]; + if(directory.length == 0) + directory = "./"; + assert(directory[$-1] == '/'); static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { - auto file = cgi.pathInfo[urlPrefix.length .. $]; - if(file.indexOf("/") != -1 || file.indexOf("\\") != -1) - return false; + auto file = decodeComponent(cgi.pathInfo[urlPrefix.length .. $]); // FIXME: is this actually correct + + if(details.recursive) { + // never allow a backslash since it isn't in a typical url anyway and makes the following checks easier + if(file.indexOf("\\") != -1) + return false; + + import std.path; + + file = std.path.buildNormalizedPath(file); + enum upOneDir = ".." ~ std.path.dirSeparator; + + // also no point doing any kind of up directory things since that makes it more likely to break out of the parent + if(file == ".." || file.startsWith(upOneDir)) + return false; + if(std.path.isAbsolute(file)) + return false; + + // FIXME: if it has slashes and stuff, should we redirect to the canonical resource? or what? + + // once it passes these filters it is probably ok. + } else { + if(file.indexOf("/") != -1 || file.indexOf("\\") != -1) + return false; + } auto contentType = contentTypeFromFileExtension(file); @@ -10874,6 +11371,7 @@ auto serveStaticFileDirectory(string urlPrefix, string directory = null) { //else if(contentType.indexOf("audio/") == 0) cgi.setCache(true); cgi.setResponseContentType(contentType); + cgi.gzipResponse = gzipRecommendedForContentType(contentType); cgi.write(std.file.read(fn), true); return true; } else { @@ -10881,7 +11379,7 @@ auto serveStaticFileDirectory(string urlPrefix, string directory = null) { } } - return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, false, DispatcherDetails(directory)); + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, false, DispatcherDetails(directory, recursive)); } /++ @@ -10959,14 +11457,162 @@ auto dispatchTo(alias handler)(string urlPrefix) { return DispatcherDefinition!(internalHandler)(urlPrefix, false); } -/+ /++ See [serveStaticFile] if you want to serve a file off disk. + + History: + Added January 28, 2023 (dub v11.0) +/ -auto serveStaticData(string urlPrefix, const(void)[] data, string contentType) { +auto serveStaticData(string urlPrefix, immutable(ubyte)[] data, string contentType, string filenameToSuggestAsDownload = null) { + assert(urlPrefix[0] == '/'); + static struct DispatcherDetails { + immutable(ubyte)[] data; + string contentType; + string filenameToSuggestAsDownload; + } + + static bool internalHandler(string urlPrefix, Cgi cgi, Object presenter, DispatcherDetails details) { + cgi.setCache(true); + cgi.setResponseContentType(details.contentType); + if(details.filenameToSuggestAsDownload.length) + cgi.header("Content-Disposition: attachment; filename=\""~details.filenameToSuggestAsDownload~"\""); + cgi.gzipResponse = gzipRecommendedForContentType(details.contentType); + cgi.write(details.data, true); + return true; + } + return DispatcherDefinition!(internalHandler, DispatcherDetails)(urlPrefix, true, DispatcherDetails(data, contentType, filenameToSuggestAsDownload)); } + +/++ + Placeholder for use with [dispatchSubsection]'s `NewPresenter` argument to indicate you want to keep the parent's presenter. + + History: + Added January 28, 2023 (dub v11.0) +/ +alias KeepExistingPresenter = typeof(null); + +/++ + For use with [dispatchSubsection]. Calls your filter with the request and if your filter returns false, + this issues the given errorCode and stops processing. + + --- + bool hasAdminPermissions(Cgi cgi) { + return true; + } + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + passFilterOrIssueError!(hasAdminPermissions, 403), + KeepExistingPresenter, + "/".serveApi!AdminFunctions + ) + ); + --- + + History: + Added January 28, 2023 (dub v11.0) ++/ +template passFilterOrIssueError(alias filter, int errorCode) { + bool passFilterOrIssueError(DispatcherDetails)(DispatcherDetails dd) { + if(filter(dd.cgi)) + return true; + dd.presenter.renderBasicError(dd.cgi, errorCode); + return false; + } +} + +/++ + Allows for a subsection of your dispatched urls to be passed through other a pre-request filter, optionally pick up an new presenter class, + and then be dispatched to their own handlers. + + --- + /+ + // a long-form filter function + bool permissionCheck(DispatcherData)(DispatcherData dd) { + // you are permitted to call mutable methods on the Cgi object + // Note if you use a Cgi subclass, you can try dynamic casting it back to your custom type to attach per-request data + // though much of the request is immutable so there's only so much you're allowed to do to modify it. + + if(checkPermissionOnRequest(dd.cgi)) { + return true; // OK, allow processing to continue + } else { + dd.presenter.renderBasicError(dd.cgi, 403); // reply forbidden to the requester + return false; // and stop further processing into this subsection + } + } + +/ + + // but you can also do short-form filters: + + bool permissionCheck(Cgi cgi) { + return ("ok" in cgi.get) !is null; + } + + // handler for the subsection + class AdminClass : WebObject { + int foo() { return 5; } + } + + // handler for the main site + class TheMainSite : WebObject {} + + mixin DispatcherMain!( + "/admin".dispatchSubsection!( + // converts our short-form filter into a long-form filter + passFilterOrIssueError!(permissionCheck, 403), + // can use a new presenter if wanted for the subsection + KeepExistingPresenter, + // and then provide child route dispatchers + "/".serveApi!AdminClass + ), + // and back to the top level + "/".serveApi!TheMainSite + ); + --- + + Note you can encapsulate sections in files like this: + + --- + auto adminDispatcher(string urlPrefix) { + return urlPrefix.dispatchSubsection!( + .... + ); + } + + mixin DispatcherMain!( + "/admin".adminDispatcher, + // and so on + ) + --- + + If you want no filter, you can pass `(cgi) => true` as the filter to approve all requests. + + If you want to keep the same presenter as the parent, use [KeepExistingPresenter] as the presenter argument. + + + History: + Added January 28, 2023 (dub v11.0) ++/ +auto dispatchSubsection(alias PreRequestFilter, NewPresenter, definitions...)(string urlPrefix) { + assert(urlPrefix[0] == '/'); + assert(urlPrefix[$-1] != '/'); + static bool internalHandler(Presenter)(string urlPrefix, Cgi cgi, Presenter presenter, const void* details) { + static if(!is(PreRequestFilter == typeof(null))) { + if(!PreRequestFilter(DispatcherData!Presenter(cgi, presenter, urlPrefix.length))) + return true; // we handled it by rejecting it + } + + static if(is(NewPresenter == Presenter) || is(NewPresenter == typeof(null))) { + return dispatcher!definitions(DispatcherData!Presenter(cgi, presenter, urlPrefix.length)); + } else { + auto newPresenter = new NewPresenter(); + return dispatcher!(definitions(DispatcherData!NewPresenter(cgi, presenter, urlPrefix.length))); + } + } + + return DispatcherDefinition!(internalHandler)(urlPrefix, false); +} /++ A URL dispatcher. @@ -10983,10 +11629,11 @@ auto serveStaticData(string urlPrefix, const(void)[] data, string contentType) { You define a series of url prefixes followed by handlers. - [dispatchTo] will send the request to another function for handling. You may want to do different pre- and post- processing there, for example, an authorization check and different page layout. You can use different - presenters and different function chains. NOT IMPLEMENTED + presenters and different function chains. See [dispatchSubsection] for details. + + [dispatchTo] will send the request to another function for handling. +/ template dispatcher(definitions...) { bool dispatcher(Presenter)(Cgi cgi, Presenter presenterArg = null) { @@ -11170,11 +11817,11 @@ bool apiDispatcher()(Cgi cgi) { version(linux) private extern(C) int eventfd (uint initval, int flags) nothrow @trusted @nogc; /* -Copyright: Adam D. Ruppe, 2008 - 2021 +Copyright: Adam D. Ruppe, 2008 - 2023 License: [http://www.boost.org/LICENSE_1_0.txt|Boost License 1.0]. Authors: Adam D. Ruppe - Copyright Adam D. Ruppe 2008 - 2021. + Copyright Adam D. Ruppe 2008 - 2023. Distributed under the Boost Software License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) |