diff --git a/.github/workflows/builds.hex.pm.yml b/.github/workflows/builds.hex.pm.yml index c3926bd044b..5fc9c7dedaf 100644 --- a/.github/workflows/builds.hex.pm.yml +++ b/.github/workflows/builds.hex.pm.yml @@ -30,7 +30,7 @@ jobs: - otp: 26 otp_version: '26.0' build_docs: build_docs - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51d9b6f75cf..d862ea1104b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,7 +15,7 @@ permissions: jobs: create_draft_release: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -40,7 +40,7 @@ jobs: - otp: 26 otp_version: '26.0' build_docs: build_docs - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 with: @@ -56,7 +56,9 @@ jobs: run: | gh release upload --clobber "${{ github.ref_name }}" \ elixir-otp-${{ matrix.otp }}.zip \ - elixir-otp-${{ matrix.otp }}.zip.sha{1,256}sum + elixir-otp-${{ matrix.otp }}.zip.sha{1,256}sum \ + elixir-otp-${{ matrix.otp }}.exe \ + elixir-otp-${{ matrix.otp }}.exe.sha{1,256}sum - name: Upload Docs to GitHub if: ${{ matrix.build_docs }} env: diff --git a/.github/workflows/release_pre_built/action.yml b/.github/workflows/release_pre_built/action.yml index faf04867ae7..dcd5c066409 100644 --- a/.github/workflows/release_pre_built/action.yml +++ b/.github/workflows/release_pre_built/action.yml @@ -22,6 +22,20 @@ runs: shasum -a 1 elixir-otp-${{ inputs.otp }}.zip > elixir-otp-${{ inputs.otp }}.zip.sha1sum shasum -a 256 elixir-otp-${{ inputs.otp }}.zip > elixir-otp-${{ inputs.otp }}.zip.sha256sum echo "$PWD/bin" >> $GITHUB_PATH + - name: Install NSIS + shell: bash + run: | + sudo apt update + sudo apt install -y nsis + - name: Build Elixir Windows Installer + shell: bash + run: | + export OTP_VERSION=${{ inputs.otp_version }} + export ELIXIR_ZIP=$PWD/elixir-otp-${{ inputs.otp }}.zip + (cd lib/elixir/scripts/windows_installer && ./build.sh) + mv lib/elixir/scripts/windows_installer/tmp/elixir-otp-${{ inputs.otp }}.exe . + shasum -a 1 elixir-otp-${{ inputs.otp }}.exe > elixir-otp-${{ inputs.otp }}.exe.sha1sum + shasum -a 256 elixir-otp-${{ inputs.otp }}.exe > elixir-otp-${{ inputs.otp }}.exe.sha256sum - name: Get latest stable ExDoc version if: ${{ inputs.build_docs }} shell: bash diff --git a/CHANGELOG.md b/CHANGELOG.md index 14b0a3b35f1..4c50abf433a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,12 +68,20 @@ In Elixir v1.15, the new reports will look like: error: undefined function foo/0 (there is no such import) my_file.exs:1 - ** (CompileError) nofile: cannot compile file (errors have been logged) + ** (CompileError) my_file.exs: cannot compile file (errors have been logged) A new function, called `Code.with_diagnostics/2`, has been added so this information can be leveraged by editors, allowing them to point to several errors at once. +### Potential incompatibilities + +As part of this effort, the behaviour where undefined variables were +transformed into nullary function calls, often leading to confusing error +reports, has been disabled during project compilation. You can invoke +`Code.compiler_options(on_undefined_variable: :warn)` at the top of +your `mix.exs` to bring the old behaviour back. + ## Integration with Erlang/OTP logger This release provides additional features such as global logger @@ -100,61 +108,185 @@ config :logger, :default_formatter, format: "$time $message $metadata" ``` -If you use `Logger.Backends.Console` or other backends, they are -still fully supported and functional. If you implement your own -backends, you want to consider migrating to +If you use `Logger.Backends.Console` with a custom device or other +backends, they are still fully supported and functional. If you +implement your own backends, you want to consider migrating to [`:logger_backends`](https://github.com/elixir-lang/logger_backends) in the long term. See the new `Logger` documentation for more information on the new features and on compatibility. -## v1.15.0-rc.1 (2022-05-29) +## v1.15.8 (2024-05-21) + +### 1. Bug fixes + +#### Elixir + + * [bin/elixir] Properly handle the `--dbg` flag in Elixir's CLI + * [System] Add a note that arguments are unsafe when invoking .bat/.com scripts on Windows via `System.cmd/3` + * [Port] Add a note that arguments are unsafe when invoking .bat/.com scripts on Windows + * [URI] Ensure `:undefined` fields are properly converted to `nil` when invoking Erlang's API + +#### Logger + + * [Logger] Ensure translators are persisted across logger restarts + +#### Mix + + * [mix compile] Ensure compile paths are accessible during compilation + +## v1.15.7 (2023-10-14) ### 1. Enhancements #### Elixir - * [File] Support distributed `File.Stream` - * [Module] Add `Module.get_last_attribute/3` - * [Task] Reduce footprint of tasks by avoiding unecessary work during spawning + * [Elixir] Allow code evaluation across Elixir versions -#### ExUnit +### 2. Bug fixes - * [ExUnit.Case] Add `ExUnit.Case.get_last_registered_test/1` +#### EEx + + * [EEx] Do not emit duplicate warnings from tokenizer + +#### Mix + + * [mix format] Correctly match file to subdirectory in `Mix.Tasks.Format.formatter_for_file/2` + +## v1.15.6 (2023-09-20) + +This release also includes fixes to the Windows installer. + +### 1. Bug fixes + +#### EEx + + * [EEx] Do not crash when printing tokenizer warnings + +#### Elixir + + * [Code] Fix formatter for nested `*` in bitstrings + * [Code] Improve feedback when an invalid block is given `Code.quoted_to_algebra/2` + * [Kernel] Trace functions before they are inlined + +#### Mix + + * [mix compile] Ensure `:extra_applications` declare in umbrella projects are loaded + * [mix deps.get] Do not check for invalid applications before deps.get + * [mix deps.update] Do not check for invalid applications before deps.update + * [mix format] Load plugins when invoking the formatter from an IDE + +## v1.15.5 (2023-08-28) + +### 1. Enhancements + +#### IEx + + * [IEx.Autocomplete] Speed up loading of struct suggestions ### 2. Bug fixes #### Elixir - * [Code] Ensure `:on_undefined_variable` option works as advertised (regression) - * [Code] Format paths in `Code.with_diagnostic/2` as relative paths (regression) - * [Kernel] Raise when macros are given to dialyzer - * [Kernel] Support bitstring specifiers as map keys in pattern (regression) - * [Module] Ensure that `Module.get_attribute/3` returns `nil` and not the given default value when an attribute has been explicitly set as `nil` - * [Task] Do not double log Task failure reports + * [Code.Fragment] Fix `Code.Fragment.surround_context/2` for aliases and submodules of non-aliases + * [Kernel] Ensure stacktrace is included when necessary when rescuing multiple exceptions in the same branch + * [Kernel] Fix index in error message for unused optional arguments #### ExUnit - * [ExUnit.CaptureLog] Allow capturing deprecated log level (regression) - * [ExUnit.DocTest] Ensure proper line is returned when failing to parse doctest results + * [ExUnit.Diff] Fix scenario where diff would not show up due to a timed-out loop #### IEx - * [IEx] Fix IO operations not returning when booting IEx (regression) + * [IEx] Force group leader to run as a binary and unicode in IEx #### Mix - * [mix deps] Ensure dependencies with `included_applications` can be loaded (regression) - * [mix format] Ensure proper formatter options are returned for files (regression) + * [mix compile] Do not assume `blake` is always available + * [mix format] Load and compile plugins if specified in subdirectories + +## v1.15.4 (2023-07-18) + +### 1. Bug fixes + +#### Mix -### 3. Soft deprecations + * [mix archive.build] Disable protocol consolidation when building archives on archive.install + * [mix compile] Track removed files per local dependency (this addresses a bug where files depending on modules from path dependencies always recompiled) + * [mix release] Do not strip relevant chunks from Erlang/OTP 26 + +## v1.15.3 (2023-07-15) + +### 1. Enhancements #### Elixir - * [Kernel] Require pin variable when accessing variable inside binary size in match + * [Kernel] Improve stacktraces when executing unnested Elixir code in a file + +#### Mix + + * [Mix] Allow to opt-out of starting apps in `Mix.install/2` -## v1.15.0-rc.0 (2022-05-22) +### 2. Bug fixes + +#### Elixir + + * [Code] Ensure `with_diagnostics` propagate warnings from inner Erlang passes + +#### IEx + + * [IEx] Fix `--remsh` on Erlang/OTP 25 and earlier + +#### Mix + + * [mix compile.elixir] Ensure `__mix_recompile__?` callbacks are properly invoked + +## v1.15.2 (2023-07-01) + +### 1. Bug fixes + +#### IEx + + * [IEx] Fix CLI being unable to boot on Windows + +## v1.15.1 (2023-06-30) + +### 1. Enhancements + + * [Code] `Code.string_to_quoted/2` honors `:static_atoms_encoder` for multi-letter sigils + +### 2. Bug fixes + +#### ExUnit + + * [ExUnit.CaptureLog] Fix race condition on concurrent `capture_log` + * [ExUnit.CaptureLog] Respect options passed to nested `capture_log` calls + * [ExUnit.Doctest] Properly compile doctests without results terminated by fences + * [ExUnit.Doctest] Allow variables defined in doctests to be used in expectation + +#### IEx + + * [IEx] Ensure `pry` works on Erlang/OTP 25 and earlier while IEx is booting + * [IEx] `Code.Fragment.surround_context` considers surround context around spaces and parens + +#### Logger + + * [Logger] Do not assume Logger has been loaded at compile-time + * [Logger.Formatter] Properly handle `:function` as metadata + +#### Mix + + * [mix compile] Ensure the current project is available on the code path after its Elixir sources are compiled + * [mix compile] Guarantee yecc/leex are available when emitting warnings from previous runs + * [mix compile] Fix bug where an external resource was deleted after its + mtime was successfully retrieved + * [mix compile] Track removed modules and exports across local deps + * [mix deps] Fix an issue where dependencies could not be started in an umbrella projects + * [mix release] Properly handle optional dependencies when there is a conflict in the application start mode + * [mix release] Remove `--werl` from release scripts on Erlang/OTP 26 + +## v1.15.0 (2023-06-19) ### 1. Enhancements @@ -175,6 +307,7 @@ new features and on compatibility. * [Date] Add `Date.before?/2` and `Date.after?/2` * [DateTime] Add `DateTime.before?/2` and `DateTime.after?/2` * [DateTime] Support precision in `DateTime.utc_now/2` + * [File] Support distributed `File.Stream` * [Inspect] `Inspect` now renders `'charlists'` as `~c"charlists"` by default * [Kernel] Break down `case` and `cond` inside `dbg/2` * [Kernel] Add `t:nonempty_binary/0` and `t:nonempty_bitstring/0` @@ -195,6 +328,8 @@ new features and on compatibility. * [NaiveDateTime] Add `NaiveDateTime.beginning_of_day/1` and `NaiveDateTime.end_of_day/1` * [NaiveDateTime] Add `NaiveDateTime.before?/2` and `NaiveDateTime.after?/2` * [NaiveDateTime] Support precision in `NaiveDateTime.utc_now/2` + * [Module] Mark functions as generated in "Docs" chunk + * [Module] Add `Module.get_last_attribute/3` * [OptionParser] Support `:return_separator` option * [Process] Add `Process.alias/0,1` and `Process.unalias/1` * [Range] Add `Range.split/2` @@ -204,6 +339,7 @@ new features and on compatibility. * [System] Support `:lines` in `System.cmd/3` to capture output line by line * [Task] Remove head of line blocking on `Task.yield_many/2` * [Task] Enable selective receive optimizations in Erlang/OTP 26+ + * [Task] Reduce tasks footprint by avoiding unecessary work during spawning * [Task.Supervisor] Do not copy args on temporary `Task.Supervisor.start_child/2` * [Time] Add `Time.before?/2` and `Time.after?/2` * [URI] Add `URI.append_path/2` @@ -212,7 +348,9 @@ new features and on compatibility. * [ExUnit] Add more color configuration to ExUnit CLI formatter * [ExUnit.Callbacks] Accept `{module, function}` tuples in ExUnit `setup` callbacks + * [ExUnit.Case] Add `ExUnit.Case.get_last_registered_test/1` * [ExUnit.Doctest] Add `ExUnit.DocTest.doctest_file/2` + * [ExUnit.Doctest] Include `doctest_data` in doctest tags * [ExUnit.Formatter] When comparing two anonymous functions, defined at the same place but capturing a different environment, we will now also diff the environments #### IEx @@ -267,16 +405,22 @@ new features and on compatibility. * [Kernel] Expand macros on the left side of -> in `try/rescue` * [Kernel] Raise on misplaced `...` inside typespecs * [Kernel] Do not import `behaviour_info` and `module_info` functions from Erlang modules + * [Kernel] Raise when macros are given to dialyzer * [Kernel.ParallelCompiler] Make sure compiler doesn't crash when there are stray messages in the inbox * [Kernel.ParallelCompiler] Track compile and runtime warnings separately + * [Module] Ensure that `Module.get_attribute/3` returns `nil` and not the given default value when an attribute has been explicitly set as `nil` * [System] Fix race condition when a script would terminate before `System.stop/1` executes + * [Task] Do not double log Task failure reports * [URI] Make sure `URI.merge/2` works accordingly with relative paths #### ExUnit * [ExUnit] Fix crash when `@tag capture_log: true` was set to true and the Logger application was shut down in the middle of the test * [ExUnit] Do not merge context as tags inside the runner to reduce memory usage when emitting events to formatters + * [ExUnit] Mark test cases as invalid when an exit occurs during `setup_all` * [ExUnit] Do not expand or collect vars from quote in ExUnit assertions + * [ExUnit.DocTest] Ensure proper line is returned when failing to parse doctest results + * [ExUnit.Doctest] Fix line information when a doctest with multiple assertions fails #### IEx @@ -288,6 +432,7 @@ new features and on compatibility. * [mix compile] Include `cwd` in compiler cache key * [mix release] Fix Windows service when invoking `erlsrv.exe` in path with spaces + * [mix xref] Raise early if `mix xref` is used at the umbrella root ### 3. Soft deprecations (no warnings emitted) @@ -295,6 +440,7 @@ new features and on compatibility. * [File] `File.cp/3` and `File.cp_r/3` with a function as third argument is deprecated in favor of a keyword list + * [Kernel] Require pin variable when accessing variable inside binary size in match * [Kernel.ParallelCompiler] Require the `:return_diagnostics` option to be set to true when compiling or requiring code diff --git a/Makefile b/Makefile index 94dc8fbe55b..724c92aee47 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PREFIX ?= /usr/local TEST_FILES ?= "*_test.exs" SHARE_PREFIX ?= $(PREFIX)/share MAN_PREFIX ?= $(SHARE_PREFIX)/man -#CANONICAL := MAJOR.MINOR/ +CANONICAL := 1.15/ CANONICAL ?= main/ DOCS_FORMAT ?= html ELIXIRC := bin/elixirc --ignore-module-conflict $(ELIXIRC_OPTS) diff --git a/RELEASE.md b/RELEASE.md index 06f906a1d67..6b224e87c09 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -14,7 +14,7 @@ 6. Copy the relevant bits from /CHANGELOG.md to the GitHub release and publish it -7. Add the release to `elixir.csv` with the minimum supported OTP version (all releases), update `erlang.csv` to the latest supported OTP version, and `_data/elixir-versions.yml` (except for RCs) files in `elixir-lang/elixir-lang.github.com` +7. Update `_data/elixir-versions.yml` (except for RCs) in `elixir-lang/elixir-lang.github.com` ## Creating a new vMAJOR.MINOR branch (after first rc) diff --git a/SECURITY.md b/SECURITY.md index 895f63dd738..7628bd3bb78 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,12 +6,11 @@ Elixir applies bug fixes only to the latest minor branch. Security patches are a Elixir version | Support :------------- | :----------------------------- -1.15 | Development -1.14 | Bug fixes and security patches +1.15 | Bug fixes and security patches +1.14 | Security patches only 1.13 | Security patches only 1.12 | Security patches only 1.11 | Security patches only -1.10 | Security patches only ## Announcements diff --git a/VERSION b/VERSION index c29540b45eb..98e863cdf81 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.15.0-rc.1 \ No newline at end of file +1.15.8 diff --git a/bin/elixir b/bin/elixir index b41c32945e9..540ff24e6d9 100755 --- a/bin/elixir +++ b/bin/elixir @@ -1,7 +1,7 @@ #!/bin/sh set -e -ELIXIR_VERSION=1.15.0-rc.1 +ELIXIR_VERSION=1.15.8 if [ $# -eq 0 ] || { [ $# -eq 1 ] && { [ "$1" = "--help" ] || [ "$1" = "-h" ]; }; }; then cat <&2 @@ -94,6 +94,7 @@ starts_with () { } ERL_EXEC="erl" +MODE="cli" I=1 E=0 LENGTH=$# @@ -104,13 +105,17 @@ while [ $I -le $LENGTH ]; do S=0 C=0 case "$1" in - +elixirc|+iex) + +elixirc) C=1 ;; - -v|--no-halt|--dbg) + +iex) C=1 + MODE="iex" ;; - -e|-r|-pr|-pa|-pz|--eval|--remsh|--dot-iex) + -v|--no-halt) + C=1 + ;; + -e|-r|-pr|-pa|-pz|--eval|--remsh|--dot-iex|--dbg) C=2 ;; --rpc-eval) @@ -223,12 +228,12 @@ fi ERTS_BIN= ERTS_BIN="$ERTS_BIN" -set -- "$ERTS_BIN$ERL_EXEC" -noshell -elixir_root "$SCRIPT_PATH"/../lib -pa "$SCRIPT_PATH"/../lib/elixir/ebin $ELIXIR_ERL_OPTIONS -s elixir start_cli $ERL "$@" +set -- "$ERTS_BIN$ERL_EXEC" -noshell -elixir_root "$SCRIPT_PATH"/../lib -pa "$SCRIPT_PATH"/../lib/elixir/ebin $ELIXIR_ERL_OPTIONS -s elixir start_$MODE $ERL "$@" if [ -n "$RUN_ERL_PIPE" ]; then ESCAPED="" for PART in "$@"; do - ESCAPED="$ESCAPED $(echo "$PART" | sed 's@[^a-zA-Z0-9_/-]@\\&@g')" + ESCAPED="$ESCAPED $(printf '%s' "$PART" | sed 's@[^a-zA-Z0-9_/-]@\\&@g')" done mkdir -p "$RUN_ERL_PIPE" mkdir -p "$RUN_ERL_LOG" diff --git a/bin/elixir.bat b/bin/elixir.bat index 282ea33727e..8fc9d0efbbf 100644 --- a/bin/elixir.bat +++ b/bin/elixir.bat @@ -1,6 +1,6 @@ @if defined ELIXIR_CLI_ECHO (@echo on) else (@echo off) -set ELIXIR_VERSION=1.15.0-rc.1 +set ELIXIR_VERSION=1.15.8 setlocal enabledelayedexpansion if ""%1""=="""" if ""%2""=="""" goto documentation @@ -81,9 +81,6 @@ set beforeExtra= rem Option which determines whether the loop is over set endLoop=0 -rem Designates which mode / Elixir component to run as -set runMode="elixir" - rem Designates the path to the current script set SCRIPT_PATH=%~dp0 @@ -106,7 +103,7 @@ if !endLoop! == 1 ( ) rem ******* EXECUTION OPTIONS ********************** if !par!=="--werl" (set useWerl=1 && goto startloop) -if !par!=="+iex" (set parsElixir=!parsElixir! +iex && goto startloop) +if !par!=="+iex" (set parsElixir=!parsElixir! +iex && set useIEx=1 && goto startloop) if !par!=="+elixirc" (set parsElixir=!parsElixir! +elixirc && goto startloop) rem ******* EVAL PARAMETERS ************************ if ""==!par:-e=! ( @@ -164,8 +161,13 @@ reg query HKCU\Console /v VirtualTerminalLevel 2>nul | findstr /e "0x1" >nul 2>n if %errorlevel% == 0 ( set beforeExtra=-elixir ansi_enabled true !beforeExtra! ) +if defined useIEx ( + set beforeExtra=-s elixir start_iex !beforeExtra! +) else ( + set beforeExtra=-s elixir start_cli !beforeExtra! +) -set beforeExtra=-noshell -elixir_root !SCRIPT_PATH!..\lib -pa !SCRIPT_PATH!..\lib\elixir\ebin -s elixir start_cli !beforeExtra! +set beforeExtra=-noshell -elixir_root "!SCRIPT_PATH!..\lib" -pa "!SCRIPT_PATH!..\lib\elixir\ebin" !beforeExtra! if defined ELIXIR_CLI_DRY_RUN ( if defined useWerl ( diff --git a/bin/iex b/bin/iex index 0840a8f9b41..5bf60509369 100755 --- a/bin/iex +++ b/bin/iex @@ -30,4 +30,4 @@ readlink_f () { SELF=$(readlink_f "$0") SCRIPT_PATH=$(dirname "$SELF") -exec "$SCRIPT_PATH"/elixir --no-halt --erl "-user elixir" -e ":elixir.start_iex()" +iex "$@" +exec "$SCRIPT_PATH"/elixir --no-halt --erl "-user elixir" +iex "$@" diff --git a/bin/iex.bat b/bin/iex.bat index 758bbf75d13..851f408c265 100644 --- a/bin/iex.bat +++ b/bin/iex.bat @@ -24,6 +24,6 @@ goto end :run if defined IEX_WITH_WERL (set __ELIXIR_IEX_FLAGS=--werl) else (set __ELIXIR_IEX_FLAGS=) -call "%~dp0\elixir.bat" --no-halt --erl "-user elixir" -e ":elixir.start_iex()" +iex %__ELIXIR_IEX_FLAGS% %* +call "%~dp0\elixir.bat" --no-halt --erl "-user elixir" +iex %__ELIXIR_IEX_FLAGS% %* :end endlocal diff --git a/lib/eex/lib/eex/compiler.ex b/lib/eex/lib/eex/compiler.ex index 24a40d66ade..de31c03eab7 100644 --- a/lib/eex/lib/eex/compiler.ex +++ b/lib/eex/lib/eex/compiler.ex @@ -71,11 +71,9 @@ defmodule EEx.Compiler do {:ok, expr, new_line, new_column, rest} -> {key, expr} = case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do - {:ok, _line, _column, warnings, tokens} -> - Enum.each(Enum.reverse(warnings), fn {location, file, msg} -> - :elixir_errors.erl_warn(location, file, msg) - end) - + {:ok, _line, _column, _warnings, tokens} -> + # We ignore warnings because the code will be tokenized + # again later with the right line+column info token_key(tokens, expr) {:error, _, _, _, _} -> diff --git a/lib/eex/test/eex/tokenizer_test.exs b/lib/eex/test/eex/tokenizer_test.exs index 09761c82ef1..f452bfbcf2e 100644 --- a/lib/eex/test/eex/tokenizer_test.exs +++ b/lib/eex/test/eex/tokenizer_test.exs @@ -5,7 +5,7 @@ defmodule EEx.TokenizerTest do @opts [indentation: 0, trim: false] - test "simple chars lists" do + test "simple charlists" do assert EEx.tokenize(~c"foo", @opts) == {:ok, [{:text, ~c"foo", %{column: 1, line: 1}}, {:eof, %{column: 4, line: 1}}]} end diff --git a/lib/eex/test/eex_test.exs b/lib/eex/test/eex_test.exs index c820a91d5a7..1e57bacb0d1 100644 --- a/lib/eex/test/eex_test.exs +++ b/lib/eex/test/eex_test.exs @@ -463,39 +463,24 @@ defmodule EExTest do end end - test "when middle expression has a modifier" do - assert ExUnit.CaptureIO.capture_io(:stderr, fn -> - EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>") - end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"] - end - - test "when end expression has a modifier" do - assert ExUnit.CaptureIO.capture_io(:stderr, fn -> - EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>") - end) =~ - ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"] - end - - test "when trying to use marker '/' without implementation" do + test "when trying to use marker '|' without implementation" do msg = - ~r/unsupported EEx syntax <%\/ %> \(the syntax is valid but not supported by the current EEx engine\)/ + ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ assert_raise EEx.SyntaxError, msg, fn -> - EEx.compile_string("<%/ true %>") + EEx.compile_string("<%| true %>") end end - test "when trying to use marker '|' without implementation" do + test "when trying to use marker '/' without implementation" do msg = - ~r/unsupported EEx syntax <%| %> \(the syntax is valid but not supported by the current EEx engine\)/ + ~r/unsupported EEx syntax <%\/ %> \(the syntax is valid but not supported by the current EEx engine\)/ assert_raise EEx.SyntaxError, msg, fn -> - EEx.compile_string("<%| true %>") + EEx.compile_string("<%/ true %>") end end - end - describe "error messages" do test "honor line numbers" do assert_raise EEx.SyntaxError, "nofile:100:6: expected closing '%>' for EEx expression", @@ -516,18 +501,30 @@ defmodule EExTest do EEx.compile_string("foo <%= bar", file: "my_file.eex") end end + end - test "when <%!-- is not closed" do - message = """ - my_file.eex:1:5: expected closing '--%>' for EEx expression - | - 1 | foo <%!-- bar - | ^\ - """ + describe "warnings" do + test "when middle expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<%= else %>false<% end %>") + end) =~ ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= else %>\"] + end - assert_raise EEx.SyntaxError, message, fn -> - EEx.compile_string("foo <%!-- bar", file: "my_file.eex") - end + test "when end expression has a modifier" do + assert ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string("foo <%= if true do %>true<% else %>false<%= end %>") + end) =~ + ~s[unexpected beginning of EEx tag \"<%=\" on \"<%= end %>\"] + end + + test "from tokenizer" do + warning = + ExUnit.CaptureIO.capture_io(:stderr, fn -> + EEx.compile_string(~s'<%= :"foo" %>', file: "tokenizer.ex") + end) + + assert warning =~ "found quoted atom \"foo\" but the quotes are not required" + assert warning =~ "tokenizer.ex:1:5" end end diff --git a/lib/elixir/lib/access.ex b/lib/elixir/lib/access.ex index 4c893b93286..a6e4569f095 100644 --- a/lib/elixir/lib/access.ex +++ b/lib/elixir/lib/access.ex @@ -191,9 +191,13 @@ defmodule Access do case __STACKTRACE__ do [unquote(top) | _] -> reason = - "#{inspect(unquote(module))} does not implement the Access behaviour. " <> - "If you are using get_in/put_in/update_in, you can specify the field " <> - "to be accessed using Access.key!/1" + """ + #{inspect(unquote(module))} does not implement the Access behaviour + + You can use the "struct.field" syntax to access struct fields. \ + You can also use Access.key!/1 to access struct fields dynamically \ + inside get_in/put_in/update_in\ + """ %{unquote(exception) | reason: reason} @@ -326,9 +330,23 @@ defmodule Access do end end + def get(list, key, _default) when is_list(list) and is_integer(key) do + raise ArgumentError, """ + the Access module does not support accessing lists by index, got: #{inspect(key)} + + Accessing a list by index is typically discouraged in Elixir, \ + instead we prefer to use the Enum module to manipulate lists \ + as a whole. If you really must access a list element by index, \ + you can Enum.at/1 or the functions in the List module\ + """ + end + def get(list, key, _default) when is_list(list) do - raise ArgumentError, - "the Access calls for keywords expect the key to be an atom, got: " <> inspect(key) + raise ArgumentError, """ + the Access module supports only keyword lists (with atom keys), got: #{inspect(key)} + + If you want to search lists of tuples, use List.keyfind/3\ + """ end def get(nil, _key, default) do @@ -377,10 +395,26 @@ defmodule Access do Map.get_and_update(map, key, fun) end - def get_and_update(list, key, fun) when is_list(list) do + def get_and_update(list, key, fun) when is_list(list) and is_atom(key) do Keyword.get_and_update(list, key, fun) end + def get_and_update(list, key, _fun) when is_list(list) and is_integer(key) do + raise ArgumentError, """ + the Access module does not support accessing lists by index, got: #{inspect(key)} + + Accessing a list by index is typically discouraged in Elixir, \ + instead we prefer to use the Enum module to manipulate lists \ + as a whole. If you really must mostify a list element by index, \ + you can Access.at/1 or the functions in the List module\ + """ + end + + def get_and_update(list, key, _fun) when is_list(list) do + raise ArgumentError, + "the Access module supports only keyword lists (with atom keys), got: " <> inspect(key) + end + def get_and_update(nil, key, _fun) do raise ArgumentError, "could not put/update key #{inspect(key)} on a nil value" end @@ -507,6 +541,21 @@ defmodule Access do iex> get_in(map, [Access.key!(:user), Access.key!(:unknown)]) ** (KeyError) key :unknown not found in: %{name: \"john\"} + The examples above could be partially written as: + + iex> map = %{user: %{name: "john"}} + iex> map.user.name + "john" + iex> get_and_update_in(map.user.name, fn prev -> + ...> {prev, String.upcase(prev)} + ...> end) + {"john", %{user: %{name: "JOHN"}}} + + However, it is not possible to remove fields using the dot notation, + as it is implified those fields must also be present. In any case, + `Access.key!/1` is useful when the key is not known in advance + and must be accessed dynamically. + An error is raised if the accessed structure is not a map/struct: iex> get_in([], [Access.key!(:foo)]) diff --git a/lib/elixir/lib/code.ex b/lib/elixir/lib/code.ex index 704ba811be9..9432b59533b 100644 --- a/lib/elixir/lib/code.ex +++ b/lib/elixir/lib/code.ex @@ -1136,7 +1136,10 @@ defmodule Code do * syntax keywords (`fn`, `do`, `else`, and so on) * atoms containing interpolation (`:"#{1 + 1} is two"`), as these - atoms are constructed at runtime. + atoms are constructed at runtime + + * atoms used to represent single-letter sigils like `:sigil_X` + (but multi-letter sigils like `:sigil_XYZ` are encoded). """ @spec string_to_quoted(List.Chars.t(), keyword) :: diff --git a/lib/elixir/lib/code/formatter.ex b/lib/elixir/lib/code/formatter.ex index 784374a4d49..e628a1320ba 100644 --- a/lib/elixir/lib/code/formatter.ex +++ b/lib/elixir/lib/code/formatter.ex @@ -431,7 +431,7 @@ defmodule Code.Formatter do {color("nil", nil, state.inspect_opts), state} end - defp quoted_to_algebra({:__block__, meta, _} = block, _context, state) do + defp quoted_to_algebra({:__block__, meta, args} = block, _context, state) when is_list(args) do {block, state} = block_to_algebra(block, line(meta), closing_line(meta), state) {surround("(", block, ")"), state} end @@ -1419,7 +1419,9 @@ defmodule Code.Formatter do defp bitstring_segment_to_algebra({{:"::", _, [segment, spec]}, i}, state, last) do {doc, state} = quoted_to_algebra(segment, :parens_arg, state) - {spec, state} = bitstring_spec_to_algebra(spec, state, state.normalize_bitstring_modifiers) + + {spec, state} = + bitstring_spec_to_algebra(spec, state, state.normalize_bitstring_modifiers, :"::") spec = wrap_in_parens_if_inspected_atom(spec) spec = if i == last, do: bitstring_wrap_parens(spec, i, last), else: spec @@ -1438,15 +1440,17 @@ defmodule Code.Formatter do {bitstring_wrap_parens(doc, i, last), state} end - defp bitstring_spec_to_algebra({op, _, [left, right]}, state, normalize_modifiers) + defp bitstring_spec_to_algebra({op, _, [left, right]}, state, normalize_modifiers, paren_op) when op in [:-, :*] do normalize_modifiers = normalize_modifiers && op != :* - {left, state} = bitstring_spec_to_algebra(left, state, normalize_modifiers) + {left, state} = bitstring_spec_to_algebra(left, state, normalize_modifiers, op) {right, state} = bitstring_spec_element_to_algebra(right, state, normalize_modifiers) - {concat(concat(left, Atom.to_string(op)), right), state} + doc = concat(concat(left, Atom.to_string(op)), right) + doc = if paren_op == :*, do: wrap_in_parens(doc), else: doc + {doc, state} end - defp bitstring_spec_to_algebra(spec, state, normalize_modifiers) do + defp bitstring_spec_to_algebra(spec, state, normalize_modifiers, _paren_op) do bitstring_spec_element_to_algebra(spec, state, normalize_modifiers) end diff --git a/lib/elixir/lib/code/fragment.ex b/lib/elixir/lib/code/fragment.ex index ddc9e31efed..3c54e91e1e6 100644 --- a/lib/elixir/lib/code/fragment.ex +++ b/lib/elixir/lib/code/fragment.ex @@ -3,10 +3,6 @@ defmodule Code.Fragment do This module provides conveniences for analyzing fragments of textual code and extract available information whenever possible. - Most of the functions in this module provide a best-effort - and may not be accurate under all circumstances. Read each - documentation for more information. - This module should be considered experimental. """ @@ -623,7 +619,7 @@ defmodule Code.Fragment do {reversed_pre, post} = adjust_position(reversed_pre, post) case take_identifier(post, []) do - {_, [], _} -> + :none -> maybe_operator(reversed_pre, post, line, opts) {:identifier, reversed_post, rest} -> @@ -631,7 +627,7 @@ defmodule Code.Fragment do reversed = reversed_post ++ reversed_pre case codepoint_cursor_context(reversed, opts) do - {{:struct, acc}, offset} -> + {{:struct, acc}, offset} when acc != [] -> build_surround({:struct, acc}, reversed, line, offset) {{:alias, acc}, offset} -> @@ -681,6 +677,9 @@ defmodule Code.Fragment do {{:alias, acc}, offset} -> build_surround({:alias, acc}, reversed, line, offset) + {{:alias, parent, acc}, offset} -> + build_surround({:alias, parent, acc}, reversed, line, offset) + {{:struct, acc}, offset} -> build_surround({:struct, acc}, reversed, line, offset) @@ -733,15 +732,31 @@ defmodule Code.Fragment do do: take_identifier(t, [h | acc]) defp take_identifier(rest, acc) do - with {[?. | t], _} <- strip_spaces(rest, 0), + {stripped, _} = strip_spaces(rest, 0) + + with [?. | t] <- stripped, {[h | _], _} when h in ?A..?Z <- strip_spaces(t, 0) do take_alias(rest, acc) else - _ -> {:identifier, acc, rest} + # Consider it an identifier if we are at the end of line + # or if we have spaces not followed by . (call) or / (arity) + _ when acc == [] and (rest == [] or (hd(rest) in @space and hd(stripped) not in ~c"/.")) -> + {:identifier, acc, rest} + + # If we are immediately followed by a container, we are still part of the identifier. + # We don't consider << as it _may_ be an operator. + _ when acc == [] and hd(stripped) in ~c"({[" -> + {:identifier, acc, rest} + + _ when acc == [] -> + :none + + _ -> + {:identifier, acc, rest} end end - defp take_alias([h | t], acc) when h not in @non_identifier, + defp take_alias([h | t], acc) when h in ?A..?Z or h in ?a..?z or h in ?0..?9 or h == ?_, do: take_alias(t, [h | acc]) defp take_alias(rest, acc) do diff --git a/lib/elixir/lib/config/provider.ex b/lib/elixir/lib/config/provider.ex index cfd7d16b199..9cc52cf72b8 100644 --- a/lib/elixir/lib/config/provider.ex +++ b/lib/elixir/lib/config/provider.ex @@ -342,7 +342,7 @@ defmodule Config.Provider do * Make the runtime value match the compile time one * Recompile your project. If the misconfigured application is a dependency, \ - you may need to run "mix deps.compile #{app} --force" + you may need to run "mix deps.clean #{app} --build" * Alternatively, you can disable this check. If you are using releases, you can \ set :validate_compile_env to false in your release configuration. If you are \ diff --git a/lib/elixir/lib/enum.ex b/lib/elixir/lib/enum.ex index b40df675ef4..edab51e69cf 100644 --- a/lib/elixir/lib/enum.ex +++ b/lib/elixir/lib/enum.ex @@ -3009,13 +3009,16 @@ defmodule Enum do if first < count and amount > 0 do amount = Kernel.min(amount, count - first) - amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + amount = amount_with_step(amount, step) fun.(first, amount, step) else [] end end + defp amount_with_step(amount, 1), do: amount + defp amount_with_step(amount, step), do: div(amount - 1, step) + 1 + @doc """ Returns a subset list of the given `enumerable`, from `start_index` (zero-based) with `amount` number of elements if available. @@ -4440,7 +4443,7 @@ defmodule Enum do if start >= 0 do amount = Kernel.min(amount, count - start) - amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + amount = amount_with_step(amount, step) fun.(start, amount, step) else [] @@ -4448,7 +4451,7 @@ defmodule Enum do end defp slice_forward(list, start, amount, step) when is_list(list) do - amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + amount = amount_with_step(amount, step) slice_list(list, start, amount, step) end @@ -4458,7 +4461,7 @@ defmodule Enum do [] {:ok, count, fun} when is_function(fun, 1) -> - amount = Kernel.min(amount, count - start) + amount = Kernel.min(amount, count - start) |> amount_with_step(step) enumerable |> fun.() |> slice_exact(start, amount, step, count) # TODO: Deprecate me in Elixir v1.18. @@ -4473,8 +4476,7 @@ defmodule Enum do end {:ok, count, fun} when is_function(fun, 3) -> - amount = Kernel.min(amount, count - start) - amount = if step == 1, do: amount, else: div(amount - 1, step) + 1 + amount = Kernel.min(amount, count - start) |> amount_with_step(step) fun.(start, amount, step) {:error, module} -> diff --git a/lib/elixir/lib/float.ex b/lib/elixir/lib/float.ex index b04187a64d0..f317b73e16c 100644 --- a/lib/elixir/lib/float.ex +++ b/lib/elixir/lib/float.ex @@ -4,6 +4,9 @@ defmodule Float do @moduledoc """ Functions for working with floating-point numbers. + For mathematical operations on top of floating-points, + see Erlang's [`:math`](`:math`) module. + ## Kernel functions There are functions related to floating-point numbers on the `Kernel` module diff --git a/lib/elixir/lib/io.ex b/lib/elixir/lib/io.ex index d055b4d86da..ec216eda34d 100644 --- a/lib/elixir/lib/io.ex +++ b/lib/elixir/lib/io.ex @@ -593,11 +593,11 @@ defmodule IO do Another example where you might want to collect a user input every new line and break on an empty line, followed by removing - redundant new line characters (`"\n"`): + redundant new line characters (`"\\n"`): IO.stream(:stdio, :line) - |> Enum.take_while(&(&1 != "\n")) - |> Enum.map(&String.replace(&1, "\n", "")) + |> Enum.take_while(&(&1 != "\\n")) + |> Enum.map(&String.replace(&1, "\\n", "")) """ @spec stream(device, :line | pos_integer) :: Enumerable.t() diff --git a/lib/elixir/lib/kernel.ex b/lib/elixir/lib/kernel.ex index 6455604fd64..9dd3a619b12 100644 --- a/lib/elixir/lib/kernel.ex +++ b/lib/elixir/lib/kernel.ex @@ -3555,10 +3555,12 @@ defmodule Kernel do not bootstrapped?(Macro) -> nil - not function? and __CALLER__.context == :match -> + not function? and (__CALLER__.context == :match or __CALLER__.context == :guard) -> raise ArgumentError, """ - invalid write attribute syntax. If you want to define an attribute, don't do this: + invalid usage of module attributes. Module attributes cannot be used inside \ + pattern matching (and guards) outside of a function. If you are trying to \ + define an attribute, do not do this: @foo = :value @@ -4901,17 +4903,6 @@ defmodule Kernel do {expanded, nil} end - # We do this so that the block is not tail-call optimized and stacktraces - # are not messed up. Basically, we just insert something between the return - # value of the block and what is returned by defmodule. Using just ":ok" or - # similar doesn't work because it's likely optimized away by the compiler. - block = - quote do - result = unquote(block) - :elixir_utils.noop() - result - end - escaped = case env do %{function: nil, lexical_tracker: pid} when is_pid(pid) -> diff --git a/lib/elixir/lib/port.ex b/lib/elixir/lib/port.ex index fefcf2fa6ab..7355294301f 100644 --- a/lib/elixir/lib/port.ex +++ b/lib/elixir/lib/port.ex @@ -79,6 +79,27 @@ defmodule Port do are for advanced usage within the VM. Also consider using `System.cmd/3` if all you want is to execute a program and retrieve its return value. + > #### Windows argument splitting and untrusted arguments {: .warning} + > + > On Unix systems, arguments are passed to a new operating system + > process as an array of strings but on Windows it is up to the child + > process to parse them and some Windows programs may apply their own + > rules, which are inconsistent with the standard C runtime `argv` parsing + > + > This is particularly troublesome when invoking `.bat` or `.com` files + > as these run implicitly through `cmd.exe`, whose argument parsing is + > vulnerable to malicious input and can be used to run arbitrary shell + > commands. + > + > Therefore, if you are running on Windows and you execute batch + > files or `.com` applications, you must not pass untrusted input as + > arguments to the program. You may avoid accidentally executing them + > by explicitly passing the extension of the program you want to run, + > such as `.exe`, and double check the program is indeed not a batch + > file or `.com` application. + > + > This affects both `spawn` and `spawn_executable`. + ### spawn The `:spawn` tuple receives a binary that is going to be executed as a diff --git a/lib/elixir/lib/regex.ex b/lib/elixir/lib/regex.ex index 9435b8c0991..5e3f87b1b0c 100644 --- a/lib/elixir/lib/regex.ex +++ b/lib/elixir/lib/regex.ex @@ -295,7 +295,7 @@ defmodule Regex do end @doc false - @deprecated "Use Kernel.is_struct/2 or pattern match on %Regex{} instead" + @deprecated "Use Kernel.is_struct(term, Regex) or pattern match on %Regex{} instead" def regex?(term) def regex?(%Regex{}), do: true def regex?(_), do: false diff --git a/lib/elixir/lib/system.ex b/lib/elixir/lib/system.ex index d2b21280842..c6104f01554 100644 --- a/lib/elixir/lib/system.ex +++ b/lib/elixir/lib/system.ex @@ -996,6 +996,25 @@ defmodule System do `Port` module describes this problem and possible solutions under the "Zombie processes" section. + > #### Windows argument splitting and untrusted arguments {: .warning} + > + > On Unix systems, arguments are passed to a new operating system + > process as an array of strings but on Windows it is up to the child + > process to parse them and some Windows programs may apply their own + > rules, which are inconsistent with the standard C runtime `argv` parsing + > + > This is particularly troublesome when invoking `.bat` or `.com` files + > as these run implicitly through `cmd.exe`, whose argument parsing is + > vulnerable to malicious input and can be used to run arbitrary shell + > commands. + > + > Therefore, if you are running on Windows and you execute batch + > files or `.com` applications, you must not pass untrusted input as + > arguments to the program. You may avoid accidentally executing them + > by explicitly passing the extension of the program you want to run, + > such as `.exe`, and double check the program is indeed not a batch + > file or `.com` application. + ## Examples iex> System.cmd("echo", ["hello"]) diff --git a/lib/elixir/lib/task/supervisor.ex b/lib/elixir/lib/task/supervisor.ex index e0c86b09b67..74bfaed5a4c 100644 --- a/lib/elixir/lib/task/supervisor.ex +++ b/lib/elixir/lib/task/supervisor.ex @@ -162,6 +162,7 @@ defmodule Task.Supervisor do * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. """ @spec async(Supervisor.supervisor(), (-> any), Keyword.t()) :: Task.t() @@ -183,6 +184,7 @@ defmodule Task.Supervisor do * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. """ @spec async(Supervisor.supervisor(), module, atom, [term], Keyword.t()) :: Task.t() @@ -208,6 +210,7 @@ defmodule Task.Supervisor do * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown or an integer indicating the timeout value, defaults to 5000 milliseconds. + The tasks must trap exits for the timeout to have an effect. ## Compatibility with OTP behaviours @@ -337,6 +340,7 @@ defmodule Task.Supervisor do * `:shutdown` - `:brutal_kill` if the tasks must be killed directly on shutdown or an integer indicating the timeout value. Defaults to `5000` milliseconds. + The tasks must trap exits for the timeout to have an effect. ## Examples @@ -455,6 +459,7 @@ defmodule Task.Supervisor do * `:shutdown` - `:brutal_kill` if the task must be killed directly on shutdown or an integer indicating the timeout value, defaults to 5000 milliseconds. + The task must trap exits for the timeout to have an effect. """ @spec start_child(Supervisor.supervisor(), (-> any), keyword) :: diff --git a/lib/elixir/lib/uri.ex b/lib/elixir/lib/uri.ex index 07ba04382bb..a2a7881beb7 100644 --- a/lib/elixir/lib/uri.ex +++ b/lib/elixir/lib/uri.ex @@ -648,16 +648,16 @@ defmodule URI do scheme = String.downcase(scheme, :ascii) case map do - %{port: port} when port != :undefined -> + %{port: port} when is_integer(port) -> %{uri | scheme: scheme} %{} -> - case default_port(scheme) do - nil -> %{uri | scheme: scheme} - port -> %{uri | scheme: scheme, port: port} - end + %{uri | scheme: scheme, port: default_port(scheme)} end + %{port: :undefined} -> + %{uri | port: nil} + %{} -> uri end diff --git a/lib/elixir/pages/compatibility-and-deprecations.md b/lib/elixir/pages/compatibility-and-deprecations.md index b79b59f00c7..86f9601b6eb 100644 --- a/lib/elixir/pages/compatibility-and-deprecations.md +++ b/lib/elixir/pages/compatibility-and-deprecations.md @@ -8,12 +8,11 @@ Elixir applies bug fixes only to the latest minor branch. Security patches are a Elixir version | Support :------------- | :----------------------------- -1.15 | Development -1.14 | Bug fixes and security patches +1.15 | Bug fixes and security patches +1.14 | Security patches only 1.13 | Security patches only 1.12 | Security patches only 1.11 | Security patches only -1.10 | Security patches only New releases are announced in the read-only [announcements mailing list](https://groups.google.com/group/elixir-lang-ann). All security releases [will be tagged with `[security]`](https://groups.google.com/forum/#!searchin/elixir-lang-ann/%5Bsecurity%5D%7Csort:date). @@ -44,7 +43,7 @@ Erlang/OTP versioning is independent from the versioning of Elixir. Erlang relea Elixir version | Supported Erlang/OTP versions :------------- | :------------------------------- 1.15 | 24 - 26 -1.14 | 23 - 25 +1.14 | 23 - 25 (and Erlang/OTP 26 from v1.14.5) 1.13 | 22 - 24 (and Erlang/OTP 25 from v1.13.4) 1.12 | 22 - 24 1.11 | 21 - 23 (and Erlang/OTP 24 from v1.11.4) @@ -82,7 +81,7 @@ Version | Deprecated feature | Replaced by (ava :-------| :-------------------------------------------------- | :--------------------------------------------------------------- [v1.15] | `Calendar.ISO.day_of_week/3` | `Calendar.ISO.day_of_week/4` (v1.11) [v1.15] | `Exception.exception?/1` | `Kernel.is_exception/1` (v1.11) -[v1.15] | `Regex.regex?/1` | `Kernel.is_struct/2` (v1.11) +[v1.15] | `Regex.regex?/1` | `Kernel.is_struct/2` (`Kernel.is_struct(term, Regex)`) (v1.11) [v1.15] | `Logger.warn/2` | `Logger.warning/2` (v1.11) [v1.14] | `use Bitwise` | `import Bitwise` (v1.0) [v1.14] | `~~~/1` | `bnot/2` (v1.0) diff --git a/lib/elixir/scripts/windows_installer/.gitignore b/lib/elixir/scripts/windows_installer/.gitignore new file mode 100644 index 00000000000..3fec32c8427 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/.gitignore @@ -0,0 +1 @@ +tmp/ diff --git a/lib/elixir/scripts/windows_installer/assets/drop.ico b/lib/elixir/scripts/windows_installer/assets/drop.ico new file mode 100644 index 00000000000..b0ba3ebae48 Binary files /dev/null and b/lib/elixir/scripts/windows_installer/assets/drop.ico differ diff --git a/lib/elixir/scripts/windows_installer/build.sh b/lib/elixir/scripts/windows_installer/build.sh new file mode 100755 index 00000000000..e9b51e91f77 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/build.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Usage: +# +# With Elixir archive: +# +# ELIXIR_ZIP=Precompiled.zip OTP_VERSION=25.3.2.2 ./build.sh +# +# With Elixir version: +# +# ELIXIR_VERSION=1.14.5 OTP_VERSION=25.3.2.2 ./build.sh +set -euo pipefail + +OTP_VERSION="${OTP_VERSION:-26.0}" +otp_release=`echo "${OTP_VERSION}" | cut -d. -f1` + +mkdir -p tmp + +ELIXIR_VERSION="${ELIXIR_VERSION:-}" +if [ -n "${ELIXIR_VERSION}" ]; then + ELIXIR_ZIP="tmp/elixir-${ELIXIR_VERSION}-otp-${otp_release}.zip" + if [ ! -f "${ELIXIR_ZIP}" ]; then + url="https://github.com/elixir-lang/elixir/releases/download/v${ELIXIR_VERSION}/elixir-otp-${otp_release}.zip" + echo "downloading ${url}" + curl --fail -L -o "${ELIXIR_ZIP}" "${url}" + fi + basename=elixir-${ELIXIR_VERSION}-otp-${otp_release} +else + basename=elixir-otp-${otp_release} +fi + +if [ ! -d "tmp/${basename}" ]; then + unzip -d "tmp/${basename}" "${ELIXIR_ZIP}" +fi + +# brew install makensis +# apt install -y nsis +# choco install -y nsis +export PATH="/c/Program Files (x86)/NSIS:${PATH}" +makensis \ + -X"OutFile tmp\\${basename}.exe" \ + -DOTP_VERSION=${OTP_VERSION} \ + -DOTP_RELEASE="${otp_release}" \ + -DELIXIR_DIR=tmp\\${basename} \ + installer.nsi + +echo "Installer path: tmp/${basename}.exe" diff --git a/lib/elixir/scripts/windows_installer/installer.nsi b/lib/elixir/scripts/windows_installer/installer.nsi new file mode 100644 index 00000000000..a9e8495ea67 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/installer.nsi @@ -0,0 +1,275 @@ +!include "MUI2.nsh" +!include "StrFunc.nsh" +${Using:StrFunc} UnStrStr + +Name "Elixir" +ManifestDPIAware true +Unicode True +InstallDir "$PROGRAMFILES64\Elixir" +!define MUI_ICON "assets\drop.ico" + +; Install Page: Install Erlang/OTP + +Page custom CheckOTPPageShow CheckOTPPageLeave + +var InstalledOTPRelease +var OTPPath + +var Dialog +var NoOTPLabel +var NoOTPLabelCreated +var OTPMismatchLabel +var OTPMismatchLabelCreated +var DownloadOTPLink +var DownloadOTPLinkCreated +var VerifyOTPButton +var VerifyOTPButtonCreated +Function CheckOTPPageShow + !insertmacro MUI_HEADER_TEXT "Checking Erlang/OTP" "" + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + Call VerifyOTP + + nsDialogs::Show +FunctionEnd + +Function VerifyOTP + ${If} $NoOTPLabelCreated == "true" + ShowWindow $NoOTPLabel ${SW_HIDE} + ${EndIf} + + ${If} $OTPMismatchLabelCreated == "true" + ShowWindow $OTPMismatchLabel ${SW_HIDE} + ${EndIf} + + ${If} $DownloadOTPLinkCreated == "true" + ShowWindow $DownloadOTPLink ${SW_HIDE} + ${Else} + StrCpy $DownloadOTPLinkCreated "true" + ${NSD_CreateLink} 0 60u 100% 20u "Download Erlang/OTP ${OTP_RELEASE}" + Pop $DownloadOTPLink + ${NSD_OnClick} $DownloadOTPLink OpenOTPDownloads + ShowWindow $DownloadOTPLink ${SW_HIDE} + ${EndIf} + + ${If} $VerifyOTPButtonCreated == "true" + ShowWindow $VerifyOTPButton ${SW_HIDE} + ${Else} + StrCpy $VerifyOTPButtonCreated "true" + ${NSD_CreateButton} 0 80u 25% 12u "Verify Erlang/OTP" + Pop $VerifyOTPButton + ${NSD_OnClick} $VerifyOTPButton VerifyOTP + ShowWindow $VerifyOTPButton ${SW_HIDE} + ${EndIf} + + StrCpy $0 0 + loop: + EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0 + StrCmp $1 "" done + ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" "" + StrCpy $OTPPath $1 + IntOp $0 $0 + 1 + goto loop + done: + + ${If} $OTPPath == "" + ${If} $NoOTPLabelCreated != "true" + StrCpy $NoOTPLabelCreated "true" + ${NSD_CreateLabel} 0 0 100% 20u "Couldn't find existing Erlang/OTP installation. Click the link below to download and install it before proceeding." + Pop $NoOTPLabel + ${EndIf} + + ShowWindow $NoOTPLabel ${SW_SHOW} + ShowWindow $DownloadOTPLink ${SW_SHOW} + ShowWindow $VerifyOTPButton ${SW_SHOW} + + ${Else} + nsExec::ExecToStack `$OTPPath\bin\erl.exe -noinput -eval "\ + io:put_chars(erlang:system_info(otp_release)),\ + halt()."` + Pop $0 + Pop $1 + + ${If} $0 == 0 + StrCpy $InstalledOTPRelease $1 + ${If} $InstalledOTPRelease == ${OTP_RELEASE} + ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath. Please proceed." + + ${Else} + ${If} $OTPMismatchLabelCreated != "true" + StrCpy $OTPMismatchLabelCreated "true" + ${NSD_CreateLabel} 0 0 100% 60u "Found existing Erlang/OTP $InstalledOTPRelease installation at $OTPPath but this Elixir installer was precompiled for Erlang/OTP ${OTP_RELEASE}. \ + $\r$\n$\r$\nYou can either search for another Elixir installer precompiled for Erlang/OTP $InstalledOTPRelease or download Erlang/OTP ${OTP_RELEASE} and install before proceeding." + Pop $OTPMismatchLabel + ${EndIf} + + ShowWindow $OTPMismatchLabel ${SW_SHOW} + ShowWindow $DownloadOTPLink ${SW_SHOW} + ShowWindow $VerifyOTPButton ${SW_SHOW} + ${EndIf} + ${Else} + SetErrorlevel 5 + MessageBox MB_ICONSTOP "Found existing Erlang/OTP installation at $OTPPath but checking it exited with $0: $1" + ${EndIf} + ${EndIf} +FunctionEnd + +Function OpenOTPDownloads + ExecShell "open" "https://www.erlang.org/downloads/${OTP_RELEASE}" +FunctionEnd + +Function CheckOTPPageLeave +FunctionEnd + +; Install Page: Files + +!insertmacro MUI_PAGE_DIRECTORY +!insertmacro MUI_PAGE_INSTFILES + +; Install Page: Finish + +Page custom FinishPageShow FinishPageLeave + +var AddOTPToPathCheckbox +var AddElixirToPathCheckbox +Function FinishPageShow + !insertmacro MUI_HEADER_TEXT "Finish Setup" "" + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + ; we add to PATH using erlang, so there must be an OTP installed to do so. + ${If} "$OTPPath" != "" + ${NSD_CreateCheckbox} 0 0 195u 10u "&Add $INSTDIR\bin to %PATH%" + Pop $AddElixirToPathCheckbox + SendMessage $AddElixirToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + + ${NSD_CreateCheckbox} 0 20u 195u 10u "&Add $OTPPath\bin to %PATH%" + Pop $AddOTPToPathCheckbox + SendMessage $AddOTPToPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + + ${NSD_CreateLabel} 0 40u 100% 20u "Note: you need to restart your shell for the environment variable changes to take effect." + ${EndIf} + + nsDialogs::Show +FunctionEnd + +var PathsToAdd +Function FinishPageLeave + ${NSD_GetState} $AddOTPToPathCheckbox $0 + ${If} $0 <> ${BST_UNCHECKED} + StrCpy $PathsToAdd ";$OTPPath\bin" + ${EndIf} + + ${NSD_GetState} $AddElixirToPathCheckbox $0 + ${If} $0 <> ${BST_UNCHECKED} + StrCpy $PathsToAdd "$PathsToAdd;$INSTDIR\bin" + ${EndIf} + + ${If} "$PathsToAdd" != "" + nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" add "$PathsToAdd"` + Pop $0 + Pop $1 + ${If} $0 != 0 + SetErrorlevel 5 + MessageBox MB_ICONSTOP "adding paths failed with $0: $1" + Quit + ${EndIf} + ${EndIf} +FunctionEnd + +Section "Install Elixir" SectionElixir + SetOutPath "$INSTDIR" + File /r "${ELIXIR_DIR}\" + File "update_system_path.erl" + + WriteUninstaller "Uninstall.exe" +SectionEnd + +; Uninstall Page: Remove from %PATH% + +var RemoveOTPFromPathCheckbox +var RemoveElixirFromPathCheckbox +Function un.FinishPageShow + !insertmacro MUI_HEADER_TEXT "Remove from %PATH%" "" + + StrCpy $0 0 + loop: + EnumRegKey $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang" $0 + StrCmp $1 "" done + ReadRegStr $1 HKLM "SOFTWARE\WOW6432NODE\Ericsson\Erlang\$1" "" + StrCpy $OTPPath $1 + IntOp $0 $0 + 1 + goto loop + done: + + nsDialogs::Create 1018 + Pop $Dialog + + ${If} $Dialog == error + Abort + ${EndIf} + + ReadRegStr $0 HKCU "Environment" "Path" + + ${UnStrStr} $1 "$0" "$INSTDIR\bin" + ${If} $1 != "" + ${NSD_CreateCheckbox} 0 0 195u 10u "&Remove $INSTDIR\bin from %PATH%" + Pop $RemoveElixirFromPathCheckbox + SendMessage $RemoveElixirFromPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + ${UnStrStr} $1 "$0" "$OTPPath\bin" + ${If} $1 != "" + ${NSD_CreateCheckbox} 0 20u 195u 10u "&Remove $OTPPath\bin from %PATH%" + Pop $RemoveOTPFromPathCheckbox + SendMessage $RemoveOTPFromPathCheckbox ${BM_SETCHECK} ${BST_CHECKED} 0 + ${EndIf} + + nsDialogs::Show +FunctionEnd + +var PathsToRemove +Function un.FinishPageLeave + ${NSD_GetState} $RemoveOTPFromPathCheckbox $1 + ${If} $1 <> ${BST_UNCHECKED} + StrCpy $PathsToRemove ";$OTPPath\bin" + ${EndIf} + + ${NSD_GetState} $RemoveElixirFromPathCheckbox $1 + ${If} $1 <> ${BST_UNCHECKED} + StrCpy $PathsToRemove "$PathsToRemove;$INSTDIR\bin" + ${EndIf} + + ${If} "$PathsToRemove" != "" + nsExec::ExecToStack `"$OTPPath\bin\escript.exe" "$INSTDIR\update_system_path.erl" remove "$PathsToRemove"` + Pop $0 + Pop $1 + ${If} $0 != 0 + SetErrorlevel 5 + MessageBox MB_ICONSTOP "removing paths failed with $0: $1" + Quit + ${EndIf} + ${EndIf} +FunctionEnd + +UninstPage custom un.FinishPageShow un.FinishPageLeave + +!insertmacro MUI_UNPAGE_DIRECTORY +!insertmacro MUI_UNPAGE_INSTFILES + +Section "Uninstall" + RMDir /r "$INSTDIR" +SectionEnd + +!insertmacro MUI_LANGUAGE "English" diff --git a/lib/elixir/scripts/windows_installer/update_system_path.erl b/lib/elixir/scripts/windows_installer/update_system_path.erl new file mode 100644 index 00000000000..6a857e01c75 --- /dev/null +++ b/lib/elixir/scripts/windows_installer/update_system_path.erl @@ -0,0 +1,42 @@ +#!/usr/bin/env escript +%%! -noinput + +%% This file is used by the Elixir installer and uninstaller. +main(["add", ";" ++ PathsToAdd]) -> + {ok, Reg} = win32reg:open([read, write]), + ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"), + {ok, SystemPath} = win32reg:value(Reg, "path"), + + NewSystemPath = + lists:foldl( + fun(Elem, Acc) -> + Elem ++ ";" ++ + binary_to_list( + iolist_to_binary( + string:replace(Acc, Elem ++ ";", "", all))) + end, + SystemPath, + string:split(PathsToAdd, ";", all) + ), + + ok = win32reg:set_value(Reg, "Path", NewSystemPath), + ok; + +main(["remove", ";" ++ PathsToRemove]) -> + {ok, Reg} = win32reg:open([read, write]), + ok = win32reg:change_key(Reg, "\\hkey_current_user\\environment"), + {ok, SystemPath} = win32reg:value(Reg, "path"), + + NewSystemPath = + lists:foldl( + fun(Elem, Acc) -> + binary_to_list( + iolist_to_binary( + string:replace(Acc, Elem ++ ";", "", all))) + end, + SystemPath, + string:split(PathsToRemove, ";", all) + ), + + ok = win32reg:set_value(Reg, "Path", NewSystemPath), + ok. diff --git a/lib/elixir/src/elixir.erl b/lib/elixir/src/elixir.erl index e24277e44f7..09c216db089 100644 --- a/lib/elixir/src/elixir.erl +++ b/lib/elixir/src/elixir.erl @@ -7,10 +7,11 @@ -export([ string_to_tokens/5, tokens_to_quoted/3, 'string_to_quoted!'/5, env_for_eval/1, quoted_to_erl/2, eval_forms/3, eval_quoted/3, - eval_quoted/4 + eval_quoted/4, eval_external_handler/3 ]). -include("elixir.hrl"). -define(system, 'Elixir.System'). +-define(elixir_eval_env, {elixir, eval_env}). %% Top level types %% TODO: Remove char_list type on v2.0 @@ -192,7 +193,8 @@ start_cli() -> 'Elixir.Kernel.CLI':main(init:get_plain_arguments()), elixir_config:booted(). -%% TODO: Delete prim_tty branches and -user on Erlang/OTP 26. +%% TODO: Delete prim_tty branches and -user on Erlang/OTP 26 +%% and add "-e :elixir.start_iex" to bin/iex instead of $MODE. start() -> case code:ensure_loaded(prim_tty) of {module, _} -> @@ -207,6 +209,7 @@ start() -> start_iex() -> case code:ensure_loaded(prim_tty) of {module, _} -> + start_cli(), spawn(fun() -> elixir_config:wait_until_booted(), (shell:whereis() =:= undefined) andalso 'Elixir.IEx':cli() @@ -355,8 +358,25 @@ eval_forms(Tree, Binding, OrigE, Opts) -> _ -> [Erl] end, - ExternalHandler = eval_external_handler(NewE), - {value, Value, NewBinding} = erl_eval:exprs(Exprs, ErlBinding, none, ExternalHandler), + ExternalHandler = eval_external_handler(), + + {value, Value, NewBinding} = + try + %% ?elixir_eval_env is used by the external handler. + %% + %% The reason why we use the process dictionary to pass the environment + %% is because we want to avoid passing closures to erl_eval, as that + %% would effectively tie the eval code to the Elixir version and it is + %% best if it depends solely on Erlang/OTP. + %% + %% The downside is that functions that escape the eval context will no + %% longer have the original environment they came from. + erlang:put(?elixir_eval_env, NewE), + erl_eval:exprs(Exprs, ErlBinding, none, ExternalHandler) + after + erlang:erase(?elixir_eval_env) + end, + PruneBefore = if Prune -> length(Binding); true -> -1 end, {DumpedBinding, DumpedVars} = @@ -367,52 +387,60 @@ eval_forms(Tree, Binding, OrigE, Opts) -> %% TODO: Remove conditional once we require Erlang/OTP 25+. -if(?OTP_RELEASE >= 25). -eval_external_handler(Env) -> - Fun = fun(Ann, FunOrModFun, Args) -> - try - case FunOrModFun of - {Mod, Fun} -> apply(Mod, Fun, Args); - Fun -> apply(Fun, Args) - end - catch - Kind:Reason:Stacktrace -> - %% Take everything up to the Elixir module - Pruned = - lists:takewhile(fun - ({elixir,_,_,_}) -> false; - (_) -> true - end, Stacktrace), - - Caller = - lists:dropwhile(fun - ({elixir,_,_,_}) -> false; - (_) -> true - end, Stacktrace), - - %% Now we prune any shared code path from erl_eval - {current_stacktrace, Current} = - erlang:process_info(self(), current_stacktrace), - - %% We need to make sure that we don't generate more - %% frames than supported. So we do our best to drop - %% from the Caller, but if the caller has no frames, - %% we need to drop from Pruned. - {DroppedCaller, ToDrop} = - case Caller of - [] -> {[], true}; - _ -> {lists:droplast(Caller), false} - end, - - Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop), - File = elixir_utils:characters_to_list(?key(Env, file)), - Location = [{file, File}, {line, erl_anno:line(Ann)}], - - %% Add file+line information at the bottom - Custom = lists:reverse([{elixir_eval, '__FILE__', 1, Location} | Reversed], DroppedCaller), - erlang:raise(Kind, Reason, Custom) +eval_external_handler() -> + {value, fun ?MODULE:eval_external_handler/3}. + +eval_external_handler(Ann, FunOrModFun, Args) -> + try + case FunOrModFun of + {Mod, Fun} -> apply(Mod, Fun, Args); + Fun -> apply(Fun, Args) end - end, - {value, Fun}. + catch + Kind:Reason:Stacktrace -> + %% Take everything up to the Elixir module + Pruned = + lists:takewhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + Caller = + lists:dropwhile(fun + ({elixir,_,_,_}) -> false; + (_) -> true + end, Stacktrace), + + %% Now we prune any shared code path from erl_eval + {current_stacktrace, Current} = + erlang:process_info(self(), current_stacktrace), + + %% We need to make sure that we don't generate more + %% frames than supported. So we do our best to drop + %% from the Caller, but if the caller has no frames, + %% we need to drop from Pruned. + {DroppedCaller, ToDrop} = + case Caller of + [] -> {[], true}; + _ -> {lists:droplast(Caller), false} + end, + + Reversed = drop_common(lists:reverse(Current), lists:reverse(Pruned), ToDrop), + + %% Add file+line information at the bottom + Bottom = + case erlang:get(?elixir_eval_env) of + #{file := File} -> + [{elixir_eval, '__FILE__', 1, + [{file, elixir_utils:characters_to_list(File)}, {line, erl_anno:line(Ann)}]}]; + + _ -> + [] + end, + + Custom = lists:reverse(Bottom ++ Reversed, DroppedCaller), + erlang:raise(Kind, Reason, Custom) + end. %% We need to check if we have dropped any frames. %% If we have not dropped frames, then we need to drop one @@ -425,8 +453,8 @@ drop_common([], [{?MODULE, _, _, _} | T2], _ToDrop) -> T2; drop_common([], [_ | T2], true) -> T2; drop_common([], T2, _) -> T2. -else. -eval_external_handler(_Env) -> - none. +eval_external_handler() -> none. +eval_external_handler(_Ann, _FunOrModFun, _Args) -> error(unused). -endif. %% Converts a quoted expression to Erlang abstract format diff --git a/lib/elixir/src/elixir_compiler.erl b/lib/elixir/src/elixir_compiler.erl index 724996112c1..487d3b3fa55 100644 --- a/lib/elixir/src/elixir_compiler.erl +++ b/lib/elixir/src/elixir_compiler.erl @@ -16,7 +16,7 @@ quoted(Forms, File, Callback) -> elixir_lexical:run( Env, - fun (LexicalEnv) -> eval_or_compile(Forms, [], LexicalEnv) end, + fun (LexicalEnv) -> maybe_fast_compile(Forms, [], LexicalEnv) end, fun (#{lexical_tracker := Pid}) -> Callback(File, Pid) end ), @@ -32,7 +32,7 @@ file(File, Callback) -> %% Evaluates the given code through the Erlang compiler. %% It may end-up evaluating the code if it is deemed a %% more efficient strategy depending on the code snippet. -eval_or_compile(Forms, Args, E) -> +maybe_fast_compile(Forms, Args, E) -> case (?key(E, module) == nil) andalso allows_fast_compilation(Forms) andalso (not elixir_config:is_bootstrap()) of true -> fast_compile(Forms, E); @@ -40,8 +40,9 @@ eval_or_compile(Forms, Args, E) -> end, ok. -compile(Quoted, ArgsList, E) -> - {Expanded, SE, EE} = elixir_expand:expand(Quoted, elixir_env:env_to_ex(E), E), +compile(Quoted, ArgsList, #{line := Line} = E) -> + Block = no_tail_optimize([{line, Line}], Quoted), + {Expanded, SE, EE} = elixir_expand:expand(Block, elixir_env:env_to_ex(E), E), elixir_env:check_unused_vars(SE, EE), {Module, Fun, Purgeable} = @@ -55,7 +56,7 @@ spawned_compile(ExExprs, #{line := Line, file := File} = E) -> {ErlExprs, _} = elixir_erl_pass:translate(ExExprs, erl_anno:new(Line), S), Module = retrieve_compiler_module(), - Fun = code_fun(?key(E, module)), + Fun = code_fun(?key(E, module)), Forms = code_mod(Fun, ErlExprs, Line, File, Module, Vars), {Module, Binary} = elixir_erl_compiler:noenv_forms(Forms, File, [nowarn_nomatch, no_bool_opt, no_ssa_opt]), @@ -106,15 +107,9 @@ allows_fast_compilation(_) -> fast_compile({'__block__', _, Exprs}, E) -> lists:foldl(fun(Expr, _) -> fast_compile(Expr, E) end, nil, Exprs); -fast_compile({defmodule, Meta, [Mod, [{do, TailBlock}]]}, NoLineE) -> +fast_compile({defmodule, Meta, [Mod, [{do, Block}]]}, NoLineE) -> E = NoLineE#{line := ?line(Meta)}, - Block = {'__block__', Meta, [ - {'=', Meta, [{result, Meta, ?MODULE}, TailBlock]}, - {{'.', Meta, [elixir_utils, noop]}, Meta, []}, - {result, Meta, ?MODULE} - ]}, - Expanded = case Mod of {'__aliases__', _, _} -> case elixir_aliases:expand_or_concat(Mod, E) of @@ -129,6 +124,13 @@ fast_compile({defmodule, Meta, [Mod, [{do, TailBlock}]]}, NoLineE) -> ContextModules = [Expanded | ?key(E, context_modules)], elixir_module:compile(Expanded, Block, [], false, E#{context_modules := ContextModules}). +no_tail_optimize(Meta, Block) -> + {'__block__', Meta, [ + {'=', Meta, [{result, Meta, ?MODULE}, Block]}, + {{'.', Meta, [elixir_utils, noop]}, Meta, []}, + {result, Meta, ?MODULE} + ]}. + %% Bootstrapper bootstrap() -> diff --git a/lib/elixir/src/elixir_dispatch.erl b/lib/elixir/src/elixir_dispatch.erl index 9d60d9f78df..5e264a4f133 100644 --- a/lib/elixir/src/elixir_dispatch.erl +++ b/lib/elixir/src/elixir_dispatch.erl @@ -130,11 +130,15 @@ dispatch_require(Meta, Receiver, Name, Args, S, E, Callback) when is_atom(Receiv case elixir_rewrite:inline(Receiver, Name, Arity) of {AR, AN} -> + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), Callback(AR, AN, Args); false -> case expand_require(Meta, Receiver, {Name, Arity}, Args, S, E) of - {ok, Receiver, Quoted} -> expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E); - error -> Callback(Receiver, Name, Args) + {ok, Receiver, Quoted} -> + expand_quoted(Meta, Receiver, Name, Arity, Quoted, S, E); + error -> + elixir_env:trace({remote_function, Meta, Receiver, Name, Arity}, E), + Callback(Receiver, Name, Args) end end; diff --git a/lib/elixir/src/elixir_erl.erl b/lib/elixir/src/elixir_erl.erl index a5d8cda83dc..41a519bf0a4 100644 --- a/lib/elixir/src/elixir_erl.erl +++ b/lib/elixir/src/elixir_erl.erl @@ -510,12 +510,15 @@ get_moduledoc_meta(Set) -> get_docs(Set, Module, Definitions, Kind) -> [{Key, - erl_anno:new(Line), + maybe_generated(erl_anno:new(Line), Ctx), [signature_to_binary(Module, Name, Signature)], doc_value(Doc, Name), Meta } || {Name, Arity} <- Definitions, - {Key, _Ctx, Line, Signature, Doc, Meta} <- ets:lookup(Set, {Kind, Name, Arity})]. + {Key, Ctx, Line, Signature, Doc, Meta} <- ets:lookup(Set, {Kind, Name, Arity})]. + +maybe_generated(Ann, nil) -> Ann; +maybe_generated(Ann, _Ctx) -> erl_anno:set_generated(true, Ann). get_callback_docs(Set, Callbacks) -> [{Key, diff --git a/lib/elixir/src/elixir_erl_compiler.erl b/lib/elixir/src/elixir_erl_compiler.erl index a299ab4c2dc..7b22b7900ac 100644 --- a/lib/elixir/src/elixir_erl_compiler.erl +++ b/lib/elixir/src/elixir_erl_compiler.erl @@ -5,25 +5,42 @@ spawn(Fun) -> CompilerInfo = get(elixir_compiler_info), + CodeDiagnostics = + case get(elixir_code_diagnostics) of + undefined -> undefined; + {_Tail, Log} -> {[], Log} + end, + {_, Ref} = spawn_monitor(fun() -> put(elixir_compiler_info, CompilerInfo), + put(elixir_code_diagnostics, CodeDiagnostics), try Fun() of - Result -> exit({ok, Result}) + Result -> exit({ok, Result, get(elixir_code_diagnostics)}) catch Kind:Reason:Stack -> - exit({Kind, Reason, Stack}) + exit({Kind, Reason, Stack, get(elixir_code_diagnostics)}) end end), receive - {'DOWN', Ref, process, _, {ok, Result}} -> + {'DOWN', Ref, process, _, {ok, Result, Diagnostics}} -> + copy_diagnostics(Diagnostics), Result; - {'DOWN', Ref, process, _, {Kind, Reason, Stack}} -> + {'DOWN', Ref, process, _, {Kind, Reason, Stack, Diagnostics}} -> + copy_diagnostics(Diagnostics), erlang:raise(Kind, Reason, Stack) end. +copy_diagnostics(undefined) -> + ok; +copy_diagnostics({Head, _}) -> + case get(elixir_code_diagnostics) of + undefined -> ok; + {Tail, Log} -> put(elixir_code_diagnostics, {Head ++ Tail, Log}) + end. + forms(Forms, File, Opts) -> compile(Forms, File, Opts ++ compile:env_compiler_options()). diff --git a/lib/elixir/src/elixir_erl_try.erl b/lib/elixir/src/elixir_erl_try.erl index 4a4deeced4c..0fd4ff80b2a 100644 --- a/lib/elixir/src/elixir_erl_try.erl +++ b/lib/elixir/src/elixir_erl_try.erl @@ -55,7 +55,7 @@ normalize_rescue(Meta, Var, Pattern, Expr, ErlangAliases) -> dynamic_normalize(Meta, Var, ?REQUIRES_STACKTRACE); false -> - case lists:splitwith(fun is_normalized_with_stacktrace/1, ErlangAliases) of + case lists:partition(fun is_normalized_with_stacktrace/1, ErlangAliases) of {[], _} -> []; {_, []} -> {'__STACKTRACE__', Meta, nil}; {Some, _} -> dynamic_normalize(Meta, Var, Some) diff --git a/lib/elixir/src/elixir_errors.erl b/lib/elixir/src/elixir_errors.erl index 5817586d22c..a10701825b0 100644 --- a/lib/elixir/src/elixir_errors.erl +++ b/lib/elixir/src/elixir_errors.erl @@ -170,13 +170,22 @@ parse_error(Location, File, <<"syntax error before: ">>, Keyword, Input) %% Produce a human-readable message for errors before a sigil parse_error(Location, File, <<"syntax error before: ">>, <<"{sigil,", _Rest/binary>> = Full, Input) -> - {sigil, _, Sigil, [Content | _], _, _, _} = parse_erl_term(Full), + {sigil, _, Atom, [Content | _], _, _, _} = parse_erl_term(Full), Content2 = case is_binary(Content) of true -> Content; false -> <<>> end, - SigilName = list_to_binary(Sigil), - Message = <<"syntax error before: sigil \~", SigilName/binary, " starting with content '", Content2/binary, "'">>, + + % :static_atoms_encoder might encode :sigil_ atoms as arbitrary terms + MaybeSigil = case is_atom(Atom) of + true -> case atom_to_binary(Atom) of + <<"sigil_", Chars/binary>> -> <<"\~", Chars/binary, " ">>; + _ -> <<>> + end; + false -> <<>> + end, + + Message = <<"syntax error before: sigil ", MaybeSigil/binary, "starting with content '", Content2/binary, "'">>, raise_snippet(Location, File, Input, 'Elixir.SyntaxError', Message); %% Binaries (and interpolation) are wrapped in [<<...>>] diff --git a/lib/elixir/src/elixir_expand.erl b/lib/elixir/src/elixir_expand.erl index dd5d5129d2d..4820bd3b3f0 100644 --- a/lib/elixir/src/elixir_expand.erl +++ b/lib/elixir/src/elixir_expand.erl @@ -856,10 +856,6 @@ expand_remote(Receiver, DotMeta, Right, Meta, Args, S, SL, #{context := Context} _ -> AttachedDotMeta = attach_context_module(Receiver, DotMeta, E), - - is_atom(Receiver) andalso - elixir_env:trace({remote_function, Meta, Receiver, Right, length(Args)}, E), - {EArgs, {SA, _}, EA} = mapfold(fun expand_arg/3, {SL, S}, E, Args), case rewrite(Context, Receiver, AttachedDotMeta, Right, Meta, EArgs, S) of diff --git a/lib/elixir/src/elixir_locals.erl b/lib/elixir/src/elixir_locals.erl index 130fa122b2a..0e8d1fbe945 100644 --- a/lib/elixir/src/elixir_locals.erl +++ b/lib/elixir/src/elixir_locals.erl @@ -118,11 +118,11 @@ format_error({function_conflict, {Receiver, {Name, Arity}}}) -> format_error({unused_args, {Name, Arity}}) -> io_lib:format("default values for the optional arguments in ~ts/~B are never used", [Name, Arity]); -format_error({unused_args, {Name, Arity}, 1}) -> - io_lib:format("the default value for the first optional argument in ~ts/~B is never used", [Name, Arity]); +format_error({unused_args, {Name, Arity}, Count}) when Arity - Count == 1 -> + io_lib:format("the default value for the last optional argument in ~ts/~B is never used", [Name, Arity]); format_error({unused_args, {Name, Arity}, Count}) -> - io_lib:format("the default values for the first ~B optional arguments in ~ts/~B are never used", [Count, Name, Arity]); + io_lib:format("the default values for the last ~B optional arguments in ~ts/~B are never used", [Arity - Count, Name, Arity]); format_error({unused_def, {Name, Arity}, defp}) -> io_lib:format("function ~ts/~B is unused", [Name, Arity]); diff --git a/lib/elixir/src/elixir_module.erl b/lib/elixir/src/elixir_module.erl index b7753708ef5..06840d301d7 100644 --- a/lib/elixir/src/elixir_module.erl +++ b/lib/elixir/src/elixir_module.erl @@ -508,19 +508,17 @@ beam_location(ModuleAsCharlist) -> checker_info() -> case get(elixir_checker_info) of undefined -> undefined; - _ -> - Log = - case erlang:get(elixir_code_diagnostics) of - {_, false} -> false; - _ -> true - end, - - {'Elixir.Module.ParallelChecker':get(), Log} + _ -> 'Elixir.Module.ParallelChecker':get() end. spawn_parallel_checker(undefined, _Module, _ModuleMap) -> - nil; -spawn_parallel_checker({CheckerInfo, Log}, Module, ModuleMap) -> + ok; +spawn_parallel_checker(CheckerInfo, Module, ModuleMap) -> + Log = + case erlang:get(elixir_code_diagnostics) of + {_, false} -> false; + _ -> true + end, 'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, Log). make_module_available(Module, Binary) -> diff --git a/lib/elixir/src/elixir_parser.yrl b/lib/elixir/src/elixir_parser.yrl index 6e48d87960a..d5341acd666 100644 --- a/lib/elixir/src/elixir_parser.yrl +++ b/lib/elixir/src/elixir_parser.yrl @@ -939,11 +939,11 @@ build_access(Expr, {List, Meta}) -> %% Interpolation aware -build_sigil({sigil, Location, Sigil, Parts, Modifiers, Indentation, Delimiter}) -> +build_sigil({sigil, Location, Atom, Parts, Modifiers, Indentation, Delimiter}) -> Meta = meta_from_location(Location), MetaWithDelimiter = [{delimiter, Delimiter} | Meta], MetaWithIndentation = meta_with_indentation(Meta, Indentation), - {list_to_atom("sigil_" ++ Sigil), + {Atom, MetaWithDelimiter, [{'<<>>', MetaWithIndentation, string_parts(Parts)}, Modifiers]}. diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 025554ea474..d060a781d69 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1030,8 +1030,8 @@ extract_heredoc_with_interpolation(Line, Column, Scope, Interpol, T, H) -> end; error -> - Message = "heredoc allows only zero or more whitespace characters followed by a new line after ", - {error, {Line, Column, io_lib:format(Message, []), [H, H, H]}} + Message = "heredoc allows only whitespace characters followed by a new line after opening ", + {error, {Line, Column + 3, io_lib:format(Message, []), [H, H, H]}} end. extract_heredoc_header("\r\n" ++ Rest) -> @@ -1550,7 +1550,7 @@ tokenize_sigil_name([S | T], NameAcc, Line, Column, Scope, Tokens) when ?is_upca tokenize_sigil_name(T, [S | NameAcc], Line, Column + 1, Scope, Tokens); % With a lowercase letter and a non-empty NameAcc we return an error. tokenize_sigil_name([S | _T] = Original, [_ | _] = NameAcc, _Line, _Column, _Scope, _Tokens) when ?is_downcase(S) -> - Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ + Message = "invalid sigil name, it should be either a one-letter lowercase letter or a" ++ " sequence of uppercase letters only, got: ", {error, Message, [$~] ++ lists:reverse(NameAcc) ++ Original}; % We finished the letters, so the name is over. @@ -1561,12 +1561,8 @@ tokenize_sigil_contents([H, H, H | T] = Original, [S | _] = SigilName, Line, Col when ?is_quote(H) -> case extract_heredoc_with_interpolation(Line, Column, Scope, ?is_downcase(S), T, H) of {ok, NewLine, NewColumn, Parts, Rest, NewScope} -> - {Final, Modifiers} = collect_modifiers(Rest, []), Indentation = NewColumn - 4, - TokenColumn = Column - 1 - length(SigilName), - Token = {sigil, {Line, TokenColumn, nil}, SigilName, Parts, Modifiers, Indentation, <>}, - NewColumnWithModifiers = NewColumn + length(Modifiers), - tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, NewScope, Tokens, Indentation, <>); {error, Reason} -> error(Reason, [$~] ++ SigilName ++ Original, Scope, Tokens) @@ -1576,12 +1572,8 @@ tokenize_sigil_contents([H | T] = Original, [S | _] = SigilName, Line, Column, S when ?is_sigil(H) -> case elixir_interpolation:extract(Line, Column + 1, Scope, ?is_downcase(S), T, sigil_terminator(H)) of {NewLine, NewColumn, Parts, Rest, NewScope} -> - {Final, Modifiers} = collect_modifiers(Rest, []), Indentation = nil, - TokenColumn = Column - 1 - length(SigilName), - Token = {sigil, {Line, TokenColumn, nil}, SigilName, tokens_to_binary(Parts), Modifiers, Indentation, <>}, - NewColumnWithModifiers = NewColumn + length(Modifiers), - tokenize(Final, NewLine, NewColumnWithModifiers, NewScope, [Token | Tokens]); + add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, tokens_to_binary(Parts), Rest, NewScope, Tokens, Indentation, <>); {error, Reason} -> Sigil = [$~, S, H], @@ -1601,6 +1593,24 @@ tokenize_sigil_contents([H | _] = Original, SigilName, Line, Column, Scope, Toke tokenize_sigil_contents([], _SigilName, Line, Column, Scope, Tokens) -> tokenize([], Line, Column, Scope, Tokens). +add_sigil_token(SigilName, Line, Column, NewLine, NewColumn, Parts, Rest, Scope, Tokens, Indentation, Delimiter) -> + TokenColumn = Column - 1 - length(SigilName), + MaybeEncoded = case SigilName of + % Single-letter sigils present no risk of atom exhaustion (limited possibilities) + [_Char] -> {ok, list_to_atom("sigil_" ++ SigilName)}; + _ -> unsafe_to_atom("sigil_" ++ SigilName, Line, TokenColumn, Scope) + end, + case MaybeEncoded of + {ok, Atom} -> + {Final, Modifiers} = collect_modifiers(Rest, []), + Token = {sigil, {Line, TokenColumn, nil}, Atom, Parts, Modifiers, Indentation, Delimiter}, + NewColumnWithModifiers = NewColumn + length(Modifiers), + tokenize(Final, NewLine, NewColumnWithModifiers, Scope, [Token | Tokens]); + + {error, Reason} -> + error(Reason, Rest, Scope, Tokens) + end. + %% Fail early on invalid do syntax. For example, after %% most keywords, after comma and so on. tokenize_keyword_terminator(DoLine, DoColumn, do, [{identifier, {Line, Column, Meta}, Atom} | T]) -> @@ -1796,7 +1806,7 @@ prune_tokens([{OpType, _, _} | _] = Tokens, [], Terminators) OpType =:= in_match_op; OpType =:= type_op; OpType =:= dual_op; OpType =:= mult_op; OpType =:= power_op; OpType =:= concat_op; OpType =:= range_op; OpType =:= xor_op; OpType =:= pipe_op; OpType =:= stab_op; OpType =:= when_op; OpType =:= assoc_op; - OpType =:= rel_op; OpType =:= ternary_op -> + OpType =:= rel_op; OpType =:= ternary_op; OpType =:= capture_op -> {Tokens, Terminators}; %%% or we traverse until the end. prune_tokens([_ | Tokens], Opener, Terminators) -> diff --git a/lib/elixir/test/elixir/code_formatter/containers_test.exs b/lib/elixir/test/elixir/code_formatter/containers_test.exs index cd8485c221f..9f3b3bc5d51 100644 --- a/lib/elixir/test/elixir/code_formatter/containers_test.exs +++ b/lib/elixir/test/elixir/code_formatter/containers_test.exs @@ -275,6 +275,8 @@ defmodule Code.Formatter.ContainersTest do assert_format "<< 1 :: 2 - unit(3) >>", "<<1::2-unit(3)>>" assert_format "<< 1 :: 2 * 3 - unit(4) >>", "<<1::2*3-unit(4)>>" assert_format "<< 1 :: 2 - unit(3) - 4 / 5 >>", "<<1::2-unit(3)-(4 / 5)>>" + assert_format "<<0 :: ( x - 1 ) * 5>>", "<<0::(x-1)*5>>" + assert_format "<<0 :: 2 * 3 * 4>>", "<<0::(2*3)*4>>" end test "in comprehensions" do diff --git a/lib/elixir/test/elixir/code_fragment_test.exs b/lib/elixir/test/elixir/code_fragment_test.exs index 7a0de0623c3..90782f7d9b1 100644 --- a/lib/elixir/test/elixir/code_fragment_test.exs +++ b/lib/elixir/test/elixir/code_fragment_test.exs @@ -428,11 +428,12 @@ defmodule CodeFragmentTest do end test "column out of range" do - assert CF.surround_context("hello", {1, 20}) == :none + assert CF.surround_context("hello", {1, 20}) == + %{begin: {1, 1}, context: {:local_or_var, ~c"hello"}, end: {1, 6}} end test "local_or_var" do - for i <- 1..8 do + for i <- 1..9 do assert CF.surround_context("hello_wo", {1, i}) == %{ context: {:local_or_var, ~c"hello_wo"}, begin: {1, 1}, @@ -440,9 +441,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo", {1, 9}) == :none + assert CF.surround_context("hello_wo ", {1, 10}) == :none - for i <- 2..9 do + for i <- 2..10 do assert CF.surround_context(" hello_wo", {1, i}) == %{ context: {:local_or_var, ~c"hello_wo"}, begin: {1, 2}, @@ -450,9 +451,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context(" hello_wo", {1, 10}) == :none + assert CF.surround_context(" hello_wo ", {1, 11}) == :none - for i <- 1..6 do + for i <- 1..7 do assert CF.surround_context("hello!", {1, i}) == %{ context: {:local_or_var, ~c"hello!"}, begin: {1, 1}, @@ -460,9 +461,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello!", {1, 7}) == :none + assert CF.surround_context("hello! ", {1, 8}) == :none - for i <- 1..5 do + for i <- 1..6 do assert CF.surround_context("안녕_세상", {1, i}) == %{ context: {:local_or_var, ~c"안녕_세상"}, begin: {1, 1}, @@ -470,7 +471,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("안녕_세상", {1, 6}) == :none + assert CF.surround_context("안녕_세상 ", {1, 6}) == :none # Keywords are not local or var for keyword <- ~w(do end after catch else rescue fn true false nil)c do @@ -484,8 +485,38 @@ defmodule CodeFragmentTest do end end - test "local call" do + test "local + operator" do for i <- 1..8 do + assert CF.surround_context("hello_wo+", {1, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo+", {1, 9}) == %{ + begin: {1, 9}, + context: {:operator, ~c"+"}, + end: {1, 10} + } + + for i <- 1..9 do + assert CF.surround_context("hello_wo +", {1, i}) == %{ + context: {:local_or_var, ~c"hello_wo"}, + begin: {1, 1}, + end: {1, 9} + } + end + + assert CF.surround_context("hello_wo +", {1, 10}) == %{ + begin: {1, 10}, + context: {:operator, ~c"+"}, + end: {1, 11} + } + end + + test "local call" do + for i <- 1..9 do assert CF.surround_context("hello_wo(", {1, i}) == %{ context: {:local_call, ~c"hello_wo"}, begin: {1, 1}, @@ -493,9 +524,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo(", {1, 9}) == :none + assert CF.surround_context("hello_wo(", {1, 10}) == :none - for i <- 1..8 do + for i <- 1..9 do assert CF.surround_context("hello_wo (", {1, i}) == %{ context: {:local_call, ~c"hello_wo"}, begin: {1, 1}, @@ -503,9 +534,10 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello_wo (", {1, 9}) == :none + assert CF.surround_context("hello_wo (", {1, 10}) == :none + assert CF.surround_context("hello_wo (", {1, 11}) == :none - for i <- 1..6 do + for i <- 1..7 do assert CF.surround_context("hello!(", {1, i}) == %{ context: {:local_call, ~c"hello!"}, begin: {1, 1}, @@ -513,9 +545,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("hello!(", {1, 7}) == :none + assert CF.surround_context("hello!(", {1, 8}) == :none - for i <- 1..5 do + for i <- 1..6 do assert CF.surround_context("안녕_세상(", {1, i}) == %{ context: {:local_call, ~c"안녕_세상"}, begin: {1, 1}, @@ -523,7 +555,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("안녕_세상(", {1, 6}) == :none + assert CF.surround_context("안녕_세상(", {1, 7}) == :none end test "local arity" do @@ -651,7 +683,7 @@ defmodule CodeFragmentTest do end test "alias" do - for i <- 1..8 do + for i <- 1..9 do assert CF.surround_context("HelloWor", {1, i}) == %{ context: {:alias, ~c"HelloWor"}, begin: {1, 1}, @@ -659,9 +691,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("HelloWor", {1, 9}) == :none + assert CF.surround_context("HelloWor ", {1, 10}) == :none - for i <- 2..9 do + for i <- 2..10 do assert CF.surround_context(" HelloWor", {1, i}) == %{ context: {:alias, ~c"HelloWor"}, begin: {1, 2}, @@ -669,9 +701,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context(" HelloWor", {1, 10}) == :none + assert CF.surround_context(" HelloWor ", {1, 11}) == :none - for i <- 1..9 do + for i <- 1..10 do assert CF.surround_context("Hello.Wor", {1, i}) == %{ context: {:alias, ~c"Hello.Wor"}, begin: {1, 1}, @@ -679,9 +711,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("Hello.Wor", {1, 10}) == :none + assert CF.surround_context("Hello.Wor ", {1, 11}) == :none - for i <- 1..11 do + for i <- 1..12 do assert CF.surround_context("Hello . Wor", {1, i}) == %{ context: {:alias, ~c"Hello.Wor"}, begin: {1, 1}, @@ -689,9 +721,9 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("Hello . Wor", {1, 12}) == :none + assert CF.surround_context("Hello . Wor ", {1, 13}) == :none - for i <- 1..15 do + for i <- 1..16 do assert CF.surround_context("Foo . Bar . Baz", {1, i}) == %{ context: {:alias, ~c"Foo.Bar.Baz"}, begin: {1, 1}, @@ -706,6 +738,14 @@ defmodule CodeFragmentTest do end: {3, 5} } end + + for i <- 1..11 do + assert CF.surround_context("Foo.Bar.Baz.foo(bar)", {1, i}) == %{ + context: {:alias, ~c"Foo.Bar.Baz"}, + begin: {1, 1}, + end: {1, 12} + } + end end test "underscored special forms" do @@ -715,17 +755,21 @@ defmodule CodeFragmentTest do end: {1, 11} } - assert CF.surround_context("__MODULE__.Foo", {1, 12}) == %{ - context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, - begin: {1, 1}, - end: {1, 15} - } + for i <- 1..15 do + assert CF.surround_context("__MODULE__.Foo", {1, i}) == %{ + context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo"}, + begin: {1, 1}, + end: {1, 15} + } + end - assert CF.surround_context("__MODULE__.Foo.Sub", {1, 16}) == %{ - context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}, - begin: {1, 1}, - end: {1, 19} - } + for i <- 1..19 do + assert CF.surround_context("__MODULE__.Foo.Sub", {1, i}) == %{ + context: {:alias, {:local_or_var, ~c"__MODULE__"}, ~c"Foo.Sub"}, + begin: {1, 1}, + end: {1, 19} + } + end assert CF.surround_context("%__MODULE__{}", {1, 5}) == %{ context: {:struct, {:local_or_var, ~c"__MODULE__"}}, @@ -771,17 +815,21 @@ defmodule CodeFragmentTest do end test "attribute submodules" do - assert CF.surround_context("@some.Foo", {1, 8}) == %{ - context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}, - begin: {1, 1}, - end: {1, 10} - } + for i <- 1..10 do + assert CF.surround_context("@some.Foo", {1, i}) == %{ + context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo"}, + begin: {1, 1}, + end: {1, 10} + } + end - assert CF.surround_context("@some.Foo.Sub", {1, 12}) == %{ - context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}, - begin: {1, 1}, - end: {1, 14} - } + for i <- 1..14 do + assert CF.surround_context("@some.Foo.Sub", {1, i}) == %{ + context: {:alias, {:module_attribute, ~c"some"}, ~c"Foo.Sub"}, + begin: {1, 1}, + end: {1, 14} + } + end assert CF.surround_context("%@some{}", {1, 5}) == %{ context: {:struct, {:module_attribute, ~c"some"}}, @@ -858,7 +906,7 @@ defmodule CodeFragmentTest do end: {1, 15} } - for i <- 2..9 do + for i <- 2..10 do assert CF.surround_context("%HelloWor", {1, i}) == %{ context: {:struct, ~c"HelloWor"}, begin: {1, 1}, @@ -866,7 +914,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("%HelloWor", {1, 10}) == :none + assert CF.surround_context("%HelloWor ", {1, 11}) == :none # With dot assert CF.surround_context("%Hello.Wor", {1, 1}) == %{ @@ -875,7 +923,7 @@ defmodule CodeFragmentTest do end: {1, 11} } - for i <- 2..10 do + for i <- 2..11 do assert CF.surround_context("%Hello.Wor", {1, i}) == %{ context: {:struct, ~c"Hello.Wor"}, begin: {1, 1}, @@ -883,7 +931,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("%Hello.Wor", {1, 11}) == :none + assert CF.surround_context("%Hello.Wor ", {1, 12}) == :none # With spaces assert CF.surround_context("% Hello . Wor", {1, 1}) == %{ @@ -892,7 +940,7 @@ defmodule CodeFragmentTest do end: {1, 14} } - for i <- 2..13 do + for i <- 2..14 do assert CF.surround_context("% Hello . Wor", {1, i}) == %{ context: {:struct, ~c"Hello.Wor"}, begin: {1, 1}, @@ -900,7 +948,7 @@ defmodule CodeFragmentTest do } end - assert CF.surround_context("% Hello . Wor", {1, 14}) == :none + assert CF.surround_context("% Hello . Wor ", {1, 15}) == :none end test "module attributes" do @@ -1098,6 +1146,8 @@ defmodule CodeFragmentTest do test "keeps operators" do assert cc2q("1 + 2") == s2q("1 + __cursor__()") + assert cc2q("&foo") == s2q("&__cursor__()") + assert cc2q("&foo/") == s2q("&foo/__cursor__()") end test "keeps function calls without parens" do @@ -1200,11 +1250,6 @@ defmodule CodeFragmentTest do assert cc2q("(fn x, y -> x + y end") == s2q("(__cursor__())") end - test "removes captures" do - assert cc2q("[& &1") == s2q("[__cursor__()]") - assert cc2q("[&(&1") == s2q("[__cursor__()]") - end - test "removes closed terminators" do assert cc2q("foo([1, 2, 3]") == s2q("foo(__cursor__())") assert cc2q("foo({1, 2, 3}") == s2q("foo(__cursor__())") diff --git a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs index a67f228bd97..6cc2fce8438 100644 --- a/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs +++ b/lib/elixir/test/elixir/code_normalizer/quoted_ast_test.exs @@ -226,6 +226,11 @@ defmodule Code.Normalizer.QuotedASTTest do assert quoted_to_string(quoted) <> "\n" == expected end + test "invalid block" do + assert quoted_to_string({:__block__, [], {:bar, [], []}}) == + "{:__block__, [], {:bar, [], []}}" + end + test "not in" do assert quoted_to_string(quote(do: false not in [])) == "false not in []" end diff --git a/lib/elixir/test/elixir/code_test.exs b/lib/elixir/test/elixir/code_test.exs index 400ed2b3be7..8879ab239e2 100644 --- a/lib/elixir/test/elixir/code_test.exs +++ b/lib/elixir/test/elixir/code_test.exs @@ -90,6 +90,28 @@ defmodule CodeTest do Code.eval_quoted(quoted, []) end) end + + test "captures unknown local calls" do + sample = """ + defmodule CodeTest.UnknownLocalCall do + def perform do + foo() + end + end + """ + + assert {:rescued, [%{message: message}]} = + Code.with_diagnostics(fn -> + try do + quoted = Code.string_to_quoted!(sample, columns: true) + Code.eval_quoted(quoted, []) + rescue + _ -> :rescued + end + end) + + assert message =~ "undefined function foo/0" + end end describe "eval_string/1,2,3" do @@ -325,7 +347,12 @@ defmodule CodeTest do assert env.versioned_vars == %{} assert_receive {:trace, {:on_module, _, _}, %{module: CodeTest.TracingPruning} = trace_env} - assert trace_env.versioned_vars == %{{:result, Kernel} => 5, {:x, nil} => 1, {:y, nil} => 4} + + assert trace_env.versioned_vars == %{ + {:result, :elixir_compiler} => 5, + {:x, nil} => 1, + {:y, nil} => 4 + } end test "with defguard" do @@ -474,6 +501,14 @@ defmodule CodeTest do :code.purge(CompileCrossSample) :code.delete(CompileCrossSample) end + + test "disables tail call optimization at the root" do + try do + Code.compile_string("List.flatten(123)") + rescue + _ -> assert Enum.any?(__STACKTRACE__, &match?({_, :__FILE__, 1, _}, &1)) + end + end end test "format_string/2 returns empty iodata for empty string" do diff --git a/lib/elixir/test/elixir/enum_test.exs b/lib/elixir/test/elixir/enum_test.exs index 066ec0e9de2..2489784a8d9 100644 --- a/lib/elixir/test/elixir/enum_test.exs +++ b/lib/elixir/test/elixir/enum_test.exs @@ -1102,6 +1102,16 @@ defmodule EnumTest do [2, 1, 3] end + test "slice on MapSets" do + assert MapSet.new(1..10) |> Enum.slice(0, 2) |> Enum.count() == 2 + assert MapSet.new(1..3) |> Enum.slice(0, 10) |> Enum.count() == 3 + assert MapSet.new(1..10) |> Enum.slice(0..1) |> Enum.count() == 2 + assert MapSet.new(1..3) |> Enum.slice(0..10) |> Enum.count() == 3 + + assert MapSet.new(1..10) |> Enum.slice(0..4//2) |> Enum.count() == 3 + assert MapSet.new(1..10) |> Enum.slice(0..5//2) |> Enum.count() == 3 + end + test "sort/1" do assert Enum.sort([5, 3, 2, 4, 1]) == [1, 2, 3, 4, 5] end diff --git a/lib/elixir/test/elixir/kernel/docs_test.exs b/lib/elixir/test/elixir/kernel/docs_test.exs index 576613aa001..c3fc35c2ab5 100644 --- a/lib/elixir/test/elixir/kernel/docs_test.exs +++ b/lib/elixir/test/elixir/kernel/docs_test.exs @@ -381,4 +381,30 @@ defmodule Kernel.DocsTest do {{:fuz, 0}, :none} ] = Enum.sort(function_docs) end + + test "generated functions are annotated as such" do + write_beam( + defmodule ToBeUsed do + defmacro __using__(_) do + quote do + @doc "Hello" + def foo, do: :bar + end + end + end + ) + + write_beam( + defmodule WillBeUsing do + use ToBeUsed + end + ) + + {:docs_v1, _, _, _, _, _, docs} = Code.fetch_docs(WillBeUsing) + + assert [ + {{:function, :foo, 0}, [generated: true, location: 399], ["foo()"], + %{"en" => "Hello"}, %{}} + ] = docs + end end diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index f7131b4f745..b436a925814 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -182,6 +182,9 @@ defmodule Kernel.ParserTest do assert Code.string_to_quoted(":there_is_no_such_atom", existing_atoms_only: true) == {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "there_is_no_such_atom"}} + + assert Code.string_to_quoted("~UNKNOWN'foo bar'", existing_atoms_only: true) == + {:error, {[line: 1, column: 1], "unsafe atom does not exist: ", "sigil_UNKNOWN"}} end test "encodes atoms" do @@ -228,6 +231,20 @@ defmodule Kernel.ParserTest do ) end + test "encodes multi-letter sigils" do + ref = make_ref() + + encoder = fn atom, meta -> + assert atom == "sigil_UNKNOWN" + assert meta[:line] == 1 + assert meta[:column] == 1 + {:ok, ref} + end + + assert {:ok, {^ref, [delimiter: "'", line: 1], [{:<<>>, [line: 1], ["abc"]}, []]}} = + Code.string_to_quoted("~UNKNOWN'abc'", static_atoms_encoder: encoder) + end + test "addresses ambiguities" do encoder = fn string, _meta -> {:ok, {:atom, string}} end @@ -254,6 +271,16 @@ defmodule Kernel.ParserTest do Code.string_to_quoted("[do: 1, true: 2, end: 3]", static_atoms_encoder: encoder) end + test "does not encode one-letter sigils" do + encoder = fn atom, _meta -> raise "shouldn't be invoked for #{atom}" end + + assert {:ok, {:sigil_z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~z'foo'", static_atoms_encoder: encoder) + + assert {:ok, {:sigil_Z, [{:delimiter, "'"}, {:line, 1}], [{:<<>>, [line: 1], ["foo"]}, []]}} = + Code.string_to_quoted("~Z'foo'", static_atoms_encoder: encoder) + end + test "returns errors on long atoms even when using static_atoms_encoder" do atom = String.duplicate("a", 256) @@ -271,6 +298,9 @@ defmodule Kernel.ParserTest do assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "there_is_no_such_atom"}} = Code.string_to_quoted(":there_is_no_such_atom", static_atoms_encoder: encoder) + + assert {:error, {[line: 1, column: 1], "Invalid atom name: ", "sigil_UNKNOWN"}} = + Code.string_to_quoted("~UNKNOWN'foo bar'", static_atoms_encoder: encoder) end test "may return tuples" do @@ -538,7 +568,7 @@ defmodule Kernel.ParserTest do describe "syntax errors" do test "invalid heredoc start" do assert_syntax_error( - ~r/nofile:1:1: heredoc allows only zero or more whitespace characters followed by a new line after \"\"\"/, + ~r/nofile:1:4: heredoc allows only whitespace characters followed by a new line after opening \"\"\"/, ~c"\"\"\"bar\n\"\"\"" ) end diff --git a/lib/elixir/test/elixir/kernel/raise_test.exs b/lib/elixir/test/elixir/kernel/raise_test.exs index f42a6b6454f..944b2a7b3c9 100644 --- a/lib/elixir/test/elixir/kernel/raise_test.exs +++ b/lib/elixir/test/elixir/kernel/raise_test.exs @@ -224,6 +224,17 @@ defmodule Kernel.RaiseTest do assert result == "an exception" end + test "named function clause (stacktrace) or runtime (no stacktrace) error" do + result = + try do + Access.get("foo", 0) + rescue + x in [FunctionClauseError, CaseClauseError] -> Exception.message(x) + end + + assert result == "no function clause matching in Access.get/3" + end + test "with higher precedence than catch" do result = try do diff --git a/lib/elixir/test/elixir/kernel/tracers_test.exs b/lib/elixir/test/elixir/kernel/tracers_test.exs index 03ffefad6a1..3116b067388 100644 --- a/lib/elixir/test/elixir/kernel/tracers_test.exs +++ b/lib/elixir/test/elixir/kernel/tracers_test.exs @@ -118,6 +118,7 @@ defmodule Kernel.TracersTest do require Integer true = Integer.is_odd(1) {1, ""} = Integer.parse("1") + "foo" = Atom.to_string(:foo) """) assert_receive {{:remote_macro, meta, Integer, :is_odd, 1}, _} @@ -127,6 +128,10 @@ defmodule Kernel.TracersTest do assert_receive {{:remote_function, meta, Integer, :parse, 1}, _} assert meta[:line] == 3 assert meta[:column] == 19 + + assert_receive {{:remote_function, meta, Atom, :to_string, 1}, _} + assert meta[:line] == 4 + assert meta[:column] == 14 end test "traces remote via captures" do diff --git a/lib/elixir/test/elixir/kernel/warning_test.exs b/lib/elixir/test/elixir/kernel/warning_test.exs index b5bb78889c8..32de8e5eb11 100644 --- a/lib/elixir/test/elixir/kernel/warning_test.exs +++ b/lib/elixir/test/elixir/kernel/warning_test.exs @@ -563,17 +563,17 @@ defmodule Kernel.WarningTest do end """) end) =~ - "the default values for the first 2 optional arguments in b/3 are never used\n nofile:3" + "the default value for the last optional argument in b/3 is never used\n nofile:3" assert capture_err(fn -> Code.eval_string(~S""" defmodule Sample3 do - def a, do: b(1) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3), do: [arg1, arg2, arg3] + def a, do: b(1, 2) + defp b(arg1, arg2 \\ 2, arg3 \\ 3, arg4 \\ 4), do: [arg1, arg2, arg3, arg4] end """) end) =~ - "the default value for the first optional argument in b/3 is never used\n nofile:3" + "the default values for the last 2 optional arguments in b/4 are never used" assert capture_err(fn -> Code.eval_string(~S""" @@ -587,17 +587,6 @@ defmodule Kernel.WarningTest do assert capture_err(fn -> Code.eval_string(~S""" defmodule Sample5 do - def a, do: b(1, 2, 3) - defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3) - - defp b(arg1, arg2, arg3), do: [arg1, arg2, arg3] - end - """) - end) =~ "default values for the optional arguments in b/3 are never used\n nofile:3" - - assert capture_err(fn -> - Code.eval_string(~S""" - defmodule Sample6 do def a, do: b(1, 2) defp b(arg1 \\ 1, arg2 \\ 2, arg3 \\ 3) @@ -605,9 +594,9 @@ defmodule Kernel.WarningTest do end """) end) =~ - "the default values for the first 2 optional arguments in b/3 are never used\n nofile:3" + "the default value for the last optional argument in b/3 is never used\n nofile:3" after - purge([Sample1, Sample2, Sample3, Sample4, Sample5, Sample6]) + purge([Sample1, Sample2, Sample3, Sample4, Sample5]) end test "unused import" do diff --git a/lib/elixir/test/elixir/kernel_test.exs b/lib/elixir/test/elixir/kernel_test.exs index ad710077bbb..72d04a403ee 100644 --- a/lib/elixir/test/elixir/kernel_test.exs +++ b/lib/elixir/test/elixir/kernel_test.exs @@ -831,11 +831,25 @@ defmodule KernelTest do end test "matching attribute" do - assert_raise ArgumentError, ~r"invalid write attribute syntax", fn -> + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> defmodule MatchAttributeInModule do @foo = 42 end end + + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> + defmodule MatchAttributeInModule do + @foo 16 + <<_::@foo>> = "ab" + end + end + + assert_raise ArgumentError, ~r"invalid usage of module attributes", fn -> + defmodule MatchAttributeInModule do + @foo 16 + <<_::size(@foo)>> = "ab" + end + end end end diff --git a/lib/elixir/test/elixir/uri_test.exs b/lib/elixir/test/elixir/uri_test.exs index e465cb323e3..8772ec97d3c 100644 --- a/lib/elixir/test/elixir/uri_test.exs +++ b/lib/elixir/test/elixir/uri_test.exs @@ -277,6 +277,32 @@ defmodule URITest do test "preserves an empty query" do assert URI.new!("http://foo.com/?").query == "" end + + test "without scheme, undefined port after host translates to nil" do + assert URI.new!("//https://www.example.com") == + %URI{ + scheme: nil, + userinfo: nil, + host: "https", + port: nil, + path: "//www.example.com", + query: nil, + fragment: nil + } + end + + test "with scheme, undefined port after host translates to nil" do + assert URI.new!("myscheme://myhost:/path/info") == + %URI{ + scheme: "myscheme", + userinfo: nil, + host: "myhost", + port: nil, + path: "/path/info", + query: nil, + fragment: nil + } + end end test "http://http://http://@http://http://?http://#http://" do diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index f3c1058770c..a4d0e8f076e 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -261,22 +261,22 @@ vc_merge_conflict_test() -> tokenize_error("<<<<<<< HEAD\n[1, 2, 3]"). sigil_terminator_test() -> - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "", nil, <<"/">>}, + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}] = tokenize("~r/foo/"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"[">>}] = tokenize("~r[foo]"), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"\"">>}] = tokenize("~r\"foo\""), + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "", nil, <<"/">>}, {comp_op, {1, 9, nil}, '=='}, {identifier, {1, 12, _}, bar}] = tokenize("~r/foo/ == bar"), - [{sigil, {1, 1, nil}, "r", [<<"foo">>], "iu", nil, <<"/">>}, + [{sigil, {1, 1, nil}, sigil_r, [<<"foo">>], "iu", nil, <<"/">>}, {comp_op, {1, 11, nil}, '=='}, {identifier, {1, 14, _}, bar}] = tokenize("~r/foo/iu == bar"), - [{sigil, {1, 1, nil}, "M", [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). + [{sigil, {1, 1, nil}, sigil_M, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8"). sigil_heredoc_test() -> - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), - [{sigil, {1, 1, nil}, "S", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), - [{sigil, {1, 1, nil}, "s", [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 0, <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''"), + [{sigil, {1, 1, nil}, sigil_S, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~S\"\"\"\n sigil heredoc\n \"\"\""), + [{sigil, {1, 1, nil}, sigil_s, [<<"sigil heredoc\n">>], "", 2, <<"\"\"\"">>}] = tokenize("~s\"\"\"\n sigil heredoc\n \"\"\""). invalid_sigil_delimiter_test() -> {1, 1, "invalid sigil delimiter: ", Message} = tokenize_error("~s\\"), diff --git a/lib/ex_unit/lib/ex_unit/callbacks.ex b/lib/ex_unit/lib/ex_unit/callbacks.ex index 060d207516a..2dbe9bd0e4a 100644 --- a/lib/ex_unit/lib/ex_unit/callbacks.ex +++ b/lib/ex_unit/lib/ex_unit/callbacks.ex @@ -549,14 +549,7 @@ defmodule ExUnit.Callbacks do end child_spec = Supervisor.child_spec(child_spec_or_module, opts) - - case Supervisor.start_child(sup, child_spec) do - {:error, {:already_started, _pid}} -> - {:error, {:duplicate_child_name, child_spec.id}} - - other -> - other - end + Supervisor.start_child(sup, child_spec) end @doc """ diff --git a/lib/ex_unit/lib/ex_unit/capture_server.ex b/lib/ex_unit/lib/ex_unit/capture_server.ex index d921ed75559..bb481a17192 100644 --- a/lib/ex_unit/lib/ex_unit/capture_server.ex +++ b/lib/ex_unit/lib/ex_unit/capture_server.ex @@ -71,11 +71,11 @@ defmodule ExUnit.CaptureServer do refs = Map.put(config.log_captures, ref, true) {level, opts} = Keyword.pop(opts, :level) - true = :ets.insert(@ets, {ref, string_io, level || :all}) + {formatter_mod, formatter_config} = Logger.default_formatter(opts) + true = :ets.insert(@ets, {ref, string_io, level || :all, formatter_mod, formatter_config}) if map_size(refs) == 1 do - formatter = Logger.default_formatter(opts) - :ok = :logger.add_handler(@name, __MODULE__, %{formatter: formatter}) + :ok = :logger.add_handler(@name, __MODULE__, %{}) status = with {:ok, config} <- :logger.get_handler_config(:default), @@ -226,13 +226,20 @@ defmodule ExUnit.CaptureServer do ## :logger handler callback. - def log(event, %{} = config) do - %{formatter: {formatter_mod, formatter_config}} = config - chardata = formatter_mod.format(event, formatter_config) - - for [string_io, level] <- :ets.match(@ets, {:_, :"$1", :"$2"}), + def log(event, _config) do + for {_ref, string_io, level, formatter_mod, formatter_config} <- :ets.tab2list(@ets), :logger.compare_levels(event.level, level) in [:gt, :eq] do - :ok = IO.write(string_io, chardata) + chardata = formatter_mod.format(event, formatter_config) + # There is a race condition where the capture_log is removed + # but another process is attempting to log to string io device + # that no longer exists, so we wrap it in try/catch. + try do + IO.write(string_io, chardata) + rescue + _ -> :ok + end end + + :ok end end diff --git a/lib/ex_unit/lib/ex_unit/case.ex b/lib/ex_unit/lib/ex_unit/case.ex index ddf4797f1f1..4a84a2dc7b3 100644 --- a/lib/ex_unit/lib/ex_unit/case.ex +++ b/lib/ex_unit/lib/ex_unit/case.ex @@ -167,6 +167,9 @@ defmodule ExUnit.Case do * `:doctest_data` - additional metadata about doctests (if a doctest) + * `:test_type` - the test type used when printing test results. + It is set by ExUnit to `:test`, `:doctest` and so on, but is customizable. + The following tags customize how tests behave: * `:capture_log` - see the "Log Capture" section below @@ -178,9 +181,6 @@ defmodule ExUnit.Case do * `:tmp_dir` - (since v1.11.0) see the "Tmp Dir" section below - * `:test_type` - the test type used when printing test results. - It is set by ExUnit to `:test`, `:doctest` and so on, but is customizable. - ## Filters Tags can also be used to identify specific tests, which can then diff --git a/lib/ex_unit/lib/ex_unit/case_template.ex b/lib/ex_unit/lib/ex_unit/case_template.ex index 5ab5b81b685..fca2ea33205 100644 --- a/lib/ex_unit/lib/ex_unit/case_template.ex +++ b/lib/ex_unit/lib/ex_unit/case_template.ex @@ -130,7 +130,7 @@ defmodule ExUnit.CaseTemplate do The second argument passed to `use MyCase` gets forwarded to `using/2` too: defmodule SomeTestCase do - use MyCase, async: true, import_helpers: true, async: true + use MyCase, async: true, import_helpers: true test "the truth" do # truth/0 comes from MyApp.TestHelpers: diff --git a/lib/ex_unit/lib/ex_unit/diff.ex b/lib/ex_unit/lib/ex_unit/diff.ex index bd8db52f6cc..051fd250ccc 100644 --- a/lib/ex_unit/lib/ex_unit/diff.ex +++ b/lib/ex_unit/lib/ex_unit/diff.ex @@ -200,11 +200,15 @@ defmodule ExUnit.Diff do {diff, env} else - diff_value(left, right, env) + non_recursive_diff_value(left, right, env) end end defp diff_value(left, right, env) do + non_recursive_diff_value(left, right, env) + end + + defp non_recursive_diff_value(left, right, env) do diff_left = escape(left) |> update_diff_meta(true) diff_right = escape(right) |> update_diff_meta(true) diff = %__MODULE__{equivalent?: false, left: diff_left, right: diff_right} diff --git a/lib/ex_unit/lib/ex_unit/doc_test.ex b/lib/ex_unit/lib/ex_unit/doc_test.ex index c6fd1affe74..beb7f768572 100644 --- a/lib/ex_unit/lib/ex_unit/doc_test.ex +++ b/lib/ex_unit/lib/ex_unit/doc_test.ex @@ -382,9 +382,15 @@ defmodule ExUnit.DocTest do last_expr = Macro.to_string(last_expr(expr_ast)) quote do + # `expr_ast` may introduce variables that may be + # used within `expected_ast` so they both need to + # unquoted together here + value = unquote(expr_ast) + expected_value = unquote(expected_ast) + ExUnit.DocTest.__test__( - unquote(expr_ast), - unquote(expected_ast), + value, + expected_value, unquote(doctest), unquote(last_expr), unquote(expected), @@ -624,57 +630,37 @@ defmodule ExUnit.DocTest do end end - defp adjust_indent(kind, [line | rest], line_no, adjusted_lines, indent, module) - when kind in [:prompt, :after_prompt] do + defp adjust_indent(kind, [line | rest], line_no, adjusted_lines, indent, module) do stripped_line = strip_indent(line, indent) + trimmed_line = String.trim_leading(line) - case String.trim_leading(line) do - "" -> - :ok + if kind != :code and trimmed_line != stripped_line do + n_spaces = if indent == 1, do: "#{indent} space", else: "#{indent} spaces" - ^stripped_line -> - :ok - - _ -> - n_spaces = if indent == 1, do: "#{indent} space", else: "#{indent} spaces" - - raise Error, - line: line_no, - module: module, - message: """ - indentation level mismatch on doctest line: #{inspect(line)} + raise Error, + line: line_no, + module: module, + message: """ + indentation level mismatch on doctest line: #{inspect(line)} - If you are planning to assert on the result of an `iex>` expression, \ - make sure the result is indented at the beginning of `iex>`, which \ - in this case is exactly #{n_spaces}. + If you are planning to assert on the result of an `iex>` expression, \ + make sure the result is indented at the beginning of `iex>`, which \ + in this case is exactly #{n_spaces}. - If instead you have an `iex>` expression that spans over multiple lines, \ - please make sure that each line after the first one begins with `...>`. - """ + If instead you have an `iex>` expression that spans over multiple lines, \ + please make sure that each line after the first one begins with `...>`. + """ end - adjusted_lines = [{adjust_prompt(stripped_line, line_no, module), line_no} | adjusted_lines] - - next = - cond do - kind == :prompt -> :after_prompt - String.starts_with?(stripped_line, @iex_prompt ++ @dot_prompt) -> :after_prompt - true -> :code - end - - adjust_indent(next, rest, line_no + 1, adjusted_lines, indent, module) - end - - defp adjust_indent(:code, [line | rest], line_no, adjusted_lines, indent, module) do - stripped_line = strip_indent(line, indent) - cond do stripped_line == "" or String.starts_with?(stripped_line, @fences) -> adjusted_lines = [{"", line_no} | adjusted_lines] adjust_indent(:text, rest, line_no + 1, adjusted_lines, 0, module) - String.starts_with?(String.trim_leading(stripped_line), @iex_prompt) -> - adjust_indent(:prompt, [line | rest], line_no, adjusted_lines, indent, module) + kind == :prompt or String.starts_with?(trimmed_line, @iex_prompt) or + (kind == :maybe_prompt and String.starts_with?(trimmed_line, @dot_prompt)) -> + line = {adjust_prompt(stripped_line, line_no, module), line_no} + adjust_indent(:maybe_prompt, rest, line_no + 1, [line | adjusted_lines], indent, module) true -> adjusted_lines = [{stripped_line, line_no} | adjusted_lines] diff --git a/lib/ex_unit/lib/ex_unit/runner.ex b/lib/ex_unit/lib/ex_unit/runner.ex index b444e30520c..3fcc6c26743 100644 --- a/lib/ex_unit/lib/ex_unit/runner.ex +++ b/lib/ex_unit/lib/ex_unit/runner.ex @@ -293,8 +293,9 @@ defmodule ExUnit.Runner do {test_module, invalid_tests, []} {:DOWN, ^module_ref, :process, ^module_pid, error} -> + invalid_tests = Enum.map(tests, &%{&1 | state: {:invalid, test_module}}) test_module = %{test_module | state: failed({:EXIT, module_pid}, error, [])} - {test_module, [], []} + {test_module, invalid_tests, []} end timeout = get_timeout(config, %{}) diff --git a/lib/ex_unit/test/ex_unit/callbacks_test.exs b/lib/ex_unit/test/ex_unit/callbacks_test.exs index f5ffe108d8a..d799fc5e352 100644 --- a/lib/ex_unit/test/ex_unit/callbacks_test.exs +++ b/lib/ex_unit/test/ex_unit/callbacks_test.exs @@ -124,6 +124,22 @@ defmodule ExUnit.CallbacksTest do "** (MatchError) no match of right hand side value: :error" end + test "doesn't choke on setup_all exits" do + defmodule SetupAllExitTest do + use ExUnit.Case + + setup_all _ do + Process.exit(self(), :error) + end + + test "ok" do + assert true + end + end + + assert capture_io(fn -> ExUnit.run() end) =~ "1 test, 0 failures, 1 invalid" + end + test "doesn't choke on dead supervisor" do defmodule StartSupervisedErrorTest do use ExUnit.Case diff --git a/lib/ex_unit/test/ex_unit/capture_log_test.exs b/lib/ex_unit/test/ex_unit/capture_log_test.exs index 5a3404ac685..d29f36926b5 100644 --- a/lib/ex_unit/test/ex_unit/capture_log_test.exs +++ b/lib/ex_unit/test/ex_unit/capture_log_test.exs @@ -114,6 +114,19 @@ defmodule ExUnit.CaptureLogTest do assert log == "id=123 | hello" end + + @tag capture_log: true + test "respect options with capture_log: true" do + options = [format: "$metadata| $message", metadata: [:id], colors: [enabled: false]] + + assert {4, log} = + with_log(options, fn -> + Logger.info("hello", id: 123) + 2 + 2 + end) + + assert log == "id=123 | hello" + end end defp wait_capture_removal() do diff --git a/lib/ex_unit/test/ex_unit/diff_test.exs b/lib/ex_unit/test/ex_unit/diff_test.exs index 941577403d7..e443c42a08b 100644 --- a/lib/ex_unit/test/ex_unit/diff_test.exs +++ b/lib/ex_unit/test/ex_unit/diff_test.exs @@ -1121,6 +1121,8 @@ defmodule ExUnit.DiffTest do ) end + @compile {:no_warn_undefined, String} + test "functions" do identity = & &1 inspect = inspect(identity) @@ -1130,11 +1132,15 @@ defmodule ExUnit.DiffTest do refute_diff(identity == :a, "-#{inspect}-", "+:a+") refute_diff({identity, identity} == :a, "-{#{inspect}, #{inspect}}", "+:a+") - refute_diff({identity, :a} == {:a, identity}, "{-#{inspect}-, -:a-}", "{+:a+, +#{inspect}+}") - refute_diff(%{identity => identity} == :a, "-%{#{inspect} => #{inspect}}", "+:a+") + refute_diff( + (&String.to_charlist/1) == (&String.unknown/1), + "-&String.to_charlist/1-", + "+&String.unknown/1" + ) + refute_diff( %Opaque{data: identity} == :a, "-#Opaque-", diff --git a/lib/ex_unit/test/ex_unit/doc_test_test.exs b/lib/ex_unit/test/ex_unit/doc_test_test.exs index 70965fa2bdf..0e9a1df144f 100644 --- a/lib/ex_unit/test/ex_unit/doc_test_test.exs +++ b/lib/ex_unit/test/ex_unit/doc_test_test.exs @@ -317,8 +317,7 @@ defmodule ExUnit.DocTestTest.FencedHeredocs do ``` ``` - iex> 1 + 2 - 3 + iex> 3 = 1 + 2 ``` ``` @@ -444,6 +443,16 @@ defmodule ExUnit.DocTestTest.Haiku do end |> ExUnit.BeamHelpers.write_beam() +defmodule ExUnit.DocTestTest.VariableInExpectation do + @doc """ + iex> num = 1 + iex> ExUnit.DocTestTest.VariableInExpectation.inc(num) + num + 1 + """ + def inc(num), do: num + 1 +end +|> ExUnit.BeamHelpers.write_beam() + defmodule ExUnit.DocTestTest.PatternMatching do def starting_line, do: __ENV__.line + 2 @@ -507,6 +516,7 @@ defmodule ExUnit.DocTestTest do doctest ExUnit.DocTestTest.IndentationHeredocs doctest ExUnit.DocTestTest.FencedHeredocs doctest ExUnit.DocTestTest.Haiku + doctest ExUnit.DocTestTest.VariableInExpectation import ExUnit.CaptureIO diff --git a/lib/ex_unit/test/ex_unit/formatter_test.exs b/lib/ex_unit/test/ex_unit/formatter_test.exs index afb66a52436..bb9642fbcc4 100644 --- a/lib/ex_unit/test/ex_unit/formatter_test.exs +++ b/lib/ex_unit/test/ex_unit/formatter_test.exs @@ -491,28 +491,25 @@ defmodule ExUnit.FormatterTest do test "inspect failure" do failure = [{:error, catch_assertion(assert :will_fail == %BadInspect{}), []}] - message = ~S''' - got FunctionClauseError with message: - - """ - no function clause matching in Inspect.ExUnit.FormatterTest.BadInspect.inspect/2 - """ - - while inspecting: - - %{__struct__: ExUnit.FormatterTest.BadInspect, key: 0} - - Stacktrace: - ''' - - assert format_test_failure(test(), failure, 1, 80, &formatter/2) =~ """ + assert format_test_failure(test(), failure, 1, 80, &formatter/2) =~ ~s''' 1) world (Hello) test/ex_unit/formatter_test.exs:1 Assertion with == failed code: assert :will_fail == %BadInspect{} left: :will_fail - right: #Inspect.Error<\n#{message}\ - """ + right: #Inspect.Error< + got FunctionClauseError with message: + + """ + no function clause matching in Inspect.ExUnit.FormatterTest.BadInspect.inspect/2 + """ + + while inspecting: + + #{inspect(%BadInspect{}, structs: false)} + + Stacktrace: + ''' end defmodule BadMessage do diff --git a/lib/ex_unit/test/ex_unit/supervised_test.exs b/lib/ex_unit/test/ex_unit/supervised_test.exs index 577ae6b9676..a02dfe3c0c5 100644 --- a/lib/ex_unit/test/ex_unit/supervised_test.exs +++ b/lib/ex_unit/test/ex_unit/supervised_test.exs @@ -73,19 +73,14 @@ defmodule ExUnit.SupervisedTest do test "starts a supervised process with ID checks" do {:ok, pid} = start_supervised({MyAgent, 0}) + assert is_pid(pid) - assert {:error, {:duplicate_child_name, ExUnit.SupervisedTest.MyAgent}} = - start_supervised({MyAgent, 0}) - - assert {:error, {{:already_started, ^pid}, _}} = start_supervised({MyAgent, 0}, id: :another) + assert {:error, _} = start_supervised({MyAgent, 0}) + assert {:error, _} = start_supervised({MyAgent, 0}, id: :another) assert_raise RuntimeError, ~r"Reason: bad child specification", fn -> start_supervised!(%{id: 1, start: :oops}) end - - assert_raise RuntimeError, ~r"Reason: already started", fn -> - start_supervised!({MyAgent, 0}, id: :another) - end end test "stops a supervised process" do diff --git a/lib/iex/lib/iex.ex b/lib/iex/lib/iex.ex index a75270bcbca..ed84b3ea36f 100644 --- a/lib/iex/lib/iex.ex +++ b/lib/iex/lib/iex.ex @@ -536,6 +536,9 @@ defmodule IEx do @doc """ Returns `true` if IEx was started, `false` otherwise. + + This means the IEx application was started, but not + that its CLI interface is running. """ @spec started?() :: boolean() def started? do @@ -579,9 +582,6 @@ defmodule IEx do @doc """ Pries into the process environment. - When you start `iex`, IEx will set this function to be the - default `dbg/2` backend unless the `--no-pry` flag is given. - This function is useful for debugging a particular chunk of code when executed by a particular process. The process becomes the evaluator of IEx commands and is temporarily changed to @@ -597,6 +597,11 @@ defmodule IEx do See also `break!/4` for others ways to pry. + > #### `dbg/0` integration + > + > By calling `iex --dbg pry`, `iex` will set this function + > as the default backend for `dbg/0` calls. + ## Examples Let's suppose you want to investigate what is happening @@ -857,6 +862,8 @@ defmodule IEx do # This is a callback invoked by Erlang shell utilities # when someone presses Ctrl+G and adds `s 'Elixir.IEx'`. + # Let's consider exposing this as iex:start() and rename + # elixir:start_iex() to iex:cli(). @doc false def start(opts \\ [], mfa \\ {IEx, :dont_display_result, []}) do # TODO: Keep only this branch, delete optional args and mfa, @@ -864,15 +871,19 @@ defmodule IEx do if Code.ensure_loaded?(:prim_tty) do spawn(fn -> {:ok, _} = Application.ensure_all_started(:iex) + :ok = :io.setopts(binary: true, encoding: :unicode) _ = for fun <- Enum.reverse(after_spawn()), do: fun.() IEx.Server.run([register: false] ++ opts) end) else spawn(fn -> - {:ok, _} = Application.ensure_all_started(:elixir) - System.wait_until_booted() - :ok = :io.setopts(binary: true, encoding: :unicode) + case :init.notify_when_started(self()) do + :started -> :ok + _ -> :init.wait_until_started() + end + {:ok, _} = Application.ensure_all_started(:iex) + :ok = :io.setopts(binary: true, encoding: :unicode) _ = for fun <- Enum.reverse(after_spawn()), do: fun.() IEx.Server.run_from_shell(opts, mfa) end) @@ -920,13 +931,18 @@ defmodule IEx do mfa end - :ok = :shell.start_interactive(shell) + case :shell.start_interactive(shell) do + :ok -> + receive do + {^ref, shell} -> shell + after + 15_000 -> + IO.puts(:stderr, "Could not start IEx CLI due to reason: :boot_timeout") + System.halt(1) + end - receive do - {^ref, shell} -> shell - after - 15_000 -> - IO.puts(:stderr, "Could not start the shell after 15 seconds, aborting...") + {:error, reason} -> + IO.puts(:stderr, "Could not start IEx CLI due to reason: #{inspect(reason)}") System.halt(1) end end diff --git a/lib/iex/lib/iex/autocomplete.ex b/lib/iex/lib/iex/autocomplete.ex index af87a16d4fe..3ea80624160 100644 --- a/lib/iex/lib/iex/autocomplete.ex +++ b/lib/iex/lib/iex/autocomplete.ex @@ -146,7 +146,7 @@ defmodule IEx.Autocomplete do @doc false def exports(mod) do - if Code.ensure_loaded?(mod) and function_exported?(mod, :__info__, 1) do + if ensure_loaded?(mod) and function_exported?(mod, :__info__, 1) do mod.__info__(:macros) ++ (mod.__info__(:functions) -- [__info__: 1]) else mod.module_info(:exports) -- [module_info: 0, module_info: 1] @@ -313,21 +313,23 @@ defmodule IEx.Autocomplete do for {alias, mod} <- aliases_from_env(shell), [name] = Module.split(alias), String.starts_with?(name, hint), - struct?(mod) and not function_exported?(mod, :exception, 1), - do: %{kind: :struct, name: name} + do: {mod, name} modules = for "Elixir." <> name = full_name <- match_modules("Elixir." <> hint, true), String.starts_with?(name, hint), mod = String.to_atom(full_name), - struct?(mod) and not function_exported?(mod, :exception, 1), - do: %{kind: :struct, name: name} + do: {mod, name} - format_expansion(aliases ++ modules, hint) - end + all = aliases ++ modules + Code.ensure_all_loaded(Enum.map(all, &elem(&1, 0))) - defp struct?(mod) do - Code.ensure_loaded?(mod) and function_exported?(mod, :__struct__, 1) + refs = + for {mod, name} <- all, + function_exported?(mod, :__struct__, 1) and not function_exported?(mod, :exception, 1), + do: %{kind: :struct, name: name} + + format_expansion(refs, hint) end defp expand_container_context(code, context, hint, shell) do @@ -420,7 +422,9 @@ defmodule IEx.Autocomplete do defp container_context_struct(cursor, pairs, aliases, shell) do with {pairs, [^cursor]} <- Enum.split(pairs, -1), alias = value_from_alias(aliases, shell), - true <- Keyword.keyword?(pairs) and struct?(alias) do + true <- + Keyword.keyword?(pairs) and ensure_loaded?(alias) and + function_exported?(alias, :__struct__, 1) do {:struct, alias, pairs} else _ -> nil diff --git a/lib/iex/lib/iex/broker.ex b/lib/iex/lib/iex/broker.ex index d3e1c720cb7..93a3f86e419 100644 --- a/lib/iex/lib/iex/broker.ex +++ b/lib/iex/lib/iex/broker.ex @@ -85,9 +85,13 @@ defmodule IEx.Broker do yes?(IO.gets(:stdio, interrupt)) end - defp yes?(string) do - is_binary(string) and String.trim(string) in ["", "y", "Y", "yes", "YES", "Yes"] - end + defp yes?(string) when is_binary(string), + do: String.trim(string) in ["", "y", "Y", "yes", "YES", "Yes"] + + defp yes?(charlist) when is_list(charlist), + do: yes?(List.to_string(charlist)) + + defp yes?(_), do: false @doc """ Client requests a takeover. diff --git a/lib/iex/lib/iex/cli.ex b/lib/iex/lib/iex/cli.ex index 9977af15fce..ffb8466b49a 100644 --- a/lib/iex/lib/iex/cli.ex +++ b/lib/iex/lib/iex/cli.ex @@ -6,7 +6,7 @@ defmodule IEx.CLI do def deprecated do if tty_works?() do - :user_drv.start([:"tty_sl -c -e", old_tty_args()]) + :user_drv.start([:"tty_sl -c -e", tty_args()]) else if get_remsh(:init.get_plain_arguments()) do IO.puts( @@ -20,14 +20,10 @@ defmodule IEx.CLI do # IEx.Broker is capable of considering all groups under user_drv but # when we use :user.start(), we need to explicitly register it instead. # If we don't register, pry doesn't work. - IEx.start([register: true] ++ options()) + IEx.start([register: true] ++ options(), {:elixir, :start_cli, []}) end end - def prompt(_n) do - [] - end - # Check if tty works. If it does not, we fall back to the # simple/dumb terminal. This is starting the linked in # driver twice, it would be nice and appropriate if we had @@ -41,7 +37,7 @@ defmodule IEx.CLI do end end - defp old_tty_args do + defp tty_args do if remote = get_remsh(:init.get_plain_arguments()) do remote = List.to_atom(append_hostname(remote)) @@ -77,7 +73,7 @@ defmodule IEx.CLI do end defp local_start_mfa do - {IEx, :start, [options()]} + {IEx, :start, [options(), {:elixir, :start_cli, []}]} end def remote_start(parent, ref) do @@ -93,8 +89,7 @@ defmodule IEx.CLI do spawn_link(fn -> receive do {:begin, ^ref, other} -> - {:ok, _} = Application.ensure_all_started(:elixir) - System.wait_until_booted() + :elixir.start_cli() send(other, {:done, ref}) end end) diff --git a/lib/iex/lib/iex/helpers.ex b/lib/iex/lib/iex/helpers.ex index 550cc51513c..f693113eb11 100644 --- a/lib/iex/lib/iex/helpers.ex +++ b/lib/iex/lib/iex/helpers.ex @@ -99,7 +99,7 @@ defmodule IEx.Helpers do reenable_tasks(config) force? = Keyword.get(options, :force, false) - args = ["--purge-consolidation-path-if-stale", consolidation] + args = ["--purge-consolidation-path-if-stale", "--return-errors", consolidation] args = if force?, do: ["--force" | args], else: args {result, _} = Mix.Task.run("compile", args) diff --git a/lib/iex/lib/iex/introspection.ex b/lib/iex/lib/iex/introspection.ex index f1c4ac51e53..278bc1834b8 100644 --- a/lib/iex/lib/iex/introspection.ex +++ b/lib/iex/lib/iex/introspection.ex @@ -293,6 +293,7 @@ defmodule IEx.Introspection do module.module_info(:exports) end |> Enum.sort() + |> Enum.dedup() result = for {^function, arity} <- exports, diff --git a/lib/iex/lib/iex/pry.ex b/lib/iex/lib/iex/pry.ex index 3cb9fe56e93..82842605f7b 100644 --- a/lib/iex/lib/iex/pry.ex +++ b/lib/iex/lib/iex/pry.ex @@ -689,6 +689,8 @@ defmodule IEx.Pry do [asts_string, :faint, " #=> ", :reset, inspect(value, options), "\n\n"] |> IO.ANSI.format() |> IO.write() + + value end defp chunk_pipeline_asts_by_line(asts, %Macro.Env{line: env_line}) do diff --git a/lib/logger/lib/logger.ex b/lib/logger/lib/logger.ex index cc84b03b05a..4dcbe3e0b5c 100644 --- a/lib/logger/lib/logger.ex +++ b/lib/logger/lib/logger.ex @@ -210,7 +210,7 @@ defmodule Logger do metadata: [:error_code, :file] Or to configure default handler, for instance, to log into a file with - built-in support for log rotation: + built-in support for log rotation and compression: config :logger, :default_handler, config: [ @@ -218,9 +218,18 @@ defmodule Logger do filesync_repeat_interval: 5000, file_check: 5000, max_no_bytes: 10_000_000, - max_no_files: 5 + max_no_files: 5, + compress_on_rotate: true ] + See [`:logger_std_h`](`:logger_std_h`) for all relevant configuration, + including overload protection. Or set `:default_handler` to false to + disable the default logging altogether: + + config :logger, :default_handler, false + + How to add new handlers is covered in later sections. + > #### Keywords or maps {: .tip} > > While Erlang's logger expects `:config` to be a map, Elixir's Logger @@ -231,14 +240,6 @@ defmodule Logger do > When reading the handler configuration using Erlang's APIs, > the configuration will always be read (and written) as a map. - See [`:logger_std_h`](`:logger_std_h`) for all relevant configuration, - including overload protection. Or set `:default_handler` to false to - disable the default logging altogether: - - config :logger, :default_handler, false - - How to add new handlers is covered in later sections. - ### Compile configuration The following configuration must be set via config files (such as @@ -329,16 +330,19 @@ defmodule Logger do default handler, but you can use Erlang's [`:logger`](`:logger`) module to add other handlers too. - Erlang/OTP handlers must be listed under your own application. For example, - to setup an additional handler that writes to disk: + Erlang/OTP handlers must be listed under your own application. + For example, to setup an additional handler, so you write to + console and file: config :my_app, :logger, [ - {:handler, :disk_log, :logger_disk_log_h, %{ + {:handler, :file_log, :logger_std_h, %{ config: %{ - file: 'system.log', + file: ~c"system.log", filesync_repeat_interval: 5000, + file_check: 5000, max_no_bytes: 10_000_000, - max_no_files: 5 + max_no_files: 5, + compress_on_rotate: true }, formatter: Logger.Formatter.new() }} @@ -355,9 +359,12 @@ defmodule Logger do flexibility but they should avoid performing any long running action in such handlers, as it may slow down the action being executed considerably. At the moment, there is no built-in overload protection for Erlang handlers, - so it is your responsibility to implement it. Alternatively, you can use the - [`:logger_backends`](https://github.com/elixir-lang/logger_backends) - project. + so it is your responsibility to implement it. + + Alternatively, you can use the + [`:logger_backends`](https://github.com/elixir-lang/logger_backends) project. + It sets up a log handler with overload protection and allows incoming events + to be dispatched to multiple backends. ## Backends and backwards compatibility @@ -824,9 +831,12 @@ defmodule Logger do defp update_translators(updater) do :elixir_config.serial(fn -> + translators = updater.(Application.fetch_env!(:logger, :translators)) + Application.put_env(:logger, :translators, translators) + with %{filters: filters} <- :logger.get_primary_config(), {{_, {fun, config}}, filters} <- List.keytake(filters, :logger_translator, 0) do - config = update_in(config.translators, updater) + config = %{config | translators: translators} :ok = :logger.set_primary_config(:filters, filters ++ [logger_translator: {fun, config}]) end end) @@ -1002,7 +1012,7 @@ defmodule Logger do compile_time_purge_matching?(compile_level, compile_metadata) -> no_log(data, quoted_metadata) - Application.fetch_env!(:logger, :always_evaluate_messages) -> + Application.get_env(:logger, :always_evaluate_messages, false) -> quote do data = Logger.__evaluate_log__(unquote(data)) metadata = unquote(quoted_metadata) @@ -1110,7 +1120,7 @@ defmodule Logger do end defp no_log(data, metadata) do - if Application.fetch_env!(:logger, :always_evaluate_messages) do + if Application.get_env(:logger, :always_evaluate_messages, false) do quote do Logger.__evaluate_log__(unquote(data)) unquote(metadata) diff --git a/lib/logger/lib/logger/formatter.ex b/lib/logger/lib/logger/formatter.ex index 3fddaaf3170..10a9515e4fb 100644 --- a/lib/logger/lib/logger/formatter.ex +++ b/lib/logger/lib/logger/formatter.ex @@ -203,7 +203,7 @@ defmodule Logger.Formatter do def format(_event, _config) do raise "invalid configuration for Logger.Formatter. " <> - "Use Logger.Formatter.init/1 to define a formatter" + "Use Logger.Formatter.new/1 to define a formatter" end defp compute_meta(:module, %{mfa: {mod, _, _}}), do: mod @@ -498,8 +498,6 @@ defmodule Logger.Formatter do rest end - defp metadata(:file, file) when is_list(file), do: file - defp metadata(:domain, [head | tail]) when is_atom(head) do Enum.map_intersperse([head | tail], ?., &Atom.to_string/1) end @@ -514,6 +512,8 @@ defmodule Logger.Formatter do Exception.format_mfa(mod, fun, arity) end + defp metadata(:function, function) when is_list(function), do: function + defp metadata(:file, file) when is_list(file), do: file defp metadata(_, list) when is_list(list), do: nil defp metadata(_, other) do diff --git a/lib/logger/test/logger/backends/handler_test.exs b/lib/logger/test/logger/backends/handler_test.exs index 26e744cdac4..92169904805 100644 --- a/lib/logger/test/logger/backends/handler_test.exs +++ b/lib/logger/test/logger/backends/handler_test.exs @@ -58,7 +58,9 @@ defmodule Logger.Backends.HandlerTest do end test "add_translator/1 and remove_translator/1 for logger formats" do + refute {CustomTranslator, :t} in Application.fetch_env!(:logger, :translators) assert Logger.add_translator({CustomTranslator, :t}) + assert {CustomTranslator, :t} in Application.fetch_env!(:logger, :translators) assert capture_log(fn -> :logger.info(~c"hello: ~p", [:ok]) diff --git a/lib/logger/test/logger/formatter_test.exs b/lib/logger/test/logger/formatter_test.exs index 2f9640c8045..82a2ac6a995 100644 --- a/lib/logger/test/logger/formatter_test.exs +++ b/lib/logger/test/logger/formatter_test.exs @@ -54,6 +54,28 @@ defmodule Logger.FormatterTest do assert format_time(time) == ["12", ?:, "30", ?:, "10", ?., [?0, "10"]] end + describe "log" do + test "handles :module and :function" do + {_, formatter} = + new( + format: "\n$time $metadata[$level] $message\n", + metadata: [:module, :function, :mfa], + colors: [enabled: false] + ) + + assert %{ + level: :warn, + msg: {:string, "foo"}, + meta: %{ + mfa: {Logger.Formatter, :compile, 1} + } + } + |> format(formatter) + |> IO.chardata_to_string() =~ + "module=Logger.Formatter function=compile/1 mfa=Logger.Formatter.compile/1" + end + end + describe "compile + format" do defmodule CompileMod do def format(_level, _msg, _ts, _md) do @@ -66,7 +88,7 @@ defmodule Logger.FormatterTest do ["\n", :time, " ", :metadata, "[", :level, "] ", :message, "\n"] end - test "compile with str" do + test "compile with string" do assert compile("$level $time $date $metadata $message $node") == Enum.intersperse([:level, :time, :date, :metadata, :message, :node], " ") diff --git a/lib/mix/lib/mix.ex b/lib/mix/lib/mix.ex index 82ba0d8864a..1eeb473aa43 100644 --- a/lib/mix/lib/mix.ex +++ b/lib/mix/lib/mix.ex @@ -600,26 +600,29 @@ defmodule Mix do """ @doc since: "1.15.0" def ensure_application!(app) when is_atom(app) do - ensure_application!(app, Mix.State.builtin_apps()) + ensure_application!(app, Mix.State.builtin_apps(), []) :ok end - defp ensure_application!(app, builtin_apps) do + defp ensure_application!(app, builtin_apps, optional) do case builtin_apps do %{^app => path} -> Code.prepend_path(path, cache: true) Application.load(app) + optional = List.wrap(Application.spec(app, :optional_applications)) Application.spec(app, :applications) |> List.wrap() - |> Enum.each(&ensure_application!(&1, builtin_apps)) + |> Enum.each(&ensure_application!(&1, builtin_apps, optional)) %{} -> - Mix.raise( - "The application \"#{app}\" could not be found. This may happen if your " <> - "Operating System broke Erlang into multiple packages and may be fixed " <> - "by installing the missing \"erlang-dev\" and \"erlang-#{app}\" packages" - ) + unless app in optional do + Mix.raise( + "The application \"#{app}\" could not be found. This may happen if your " <> + "Operating System broke Erlang into multiple packages and may be fixed " <> + "by installing the missing \"erlang-dev\" and \"erlang-#{app}\" packages" + ) + end end end @@ -674,6 +677,9 @@ defmodule Mix do * `:lockfile` (since v1.14.0) - path to a lockfile to be used as a basis of dependency resolution. + * `:start_applications` (since v1.15.3) - if `true`, ensures that installed app + and its dependencies are started after install (Default: `true`) + ## Examples Installing `:decimal` and `:jason`: @@ -794,6 +800,7 @@ defmodule Mix do config_path = expand_path(opts[:config_path], deps, :config_path, "config/config.exs") system_env = Keyword.get(opts, :system_env, []) consolidate_protocols? = Keyword.get(opts, :consolidate_protocols, true) + start_applications? = Keyword.get(opts, :start_applications, true) id = {deps, config, system_env, consolidate_protocols?} @@ -893,9 +900,11 @@ defmodule Mix do end end) - for %{app: app, opts: opts} <- Mix.Dep.cached(), - Keyword.get(opts, :runtime, true) and Keyword.get(opts, :app, true) do - Application.ensure_all_started(app) + if start_applications? do + for %{app: app, opts: opts} <- Mix.Dep.cached(), + Keyword.get(opts, :runtime, true) and Keyword.get(opts, :app, true) do + Application.ensure_all_started(app) + end end Mix.State.put(:installed, id) diff --git a/lib/mix/lib/mix/app_loader.ex b/lib/mix/lib/mix/app_loader.ex index 2b0b4926b84..363e2f3ceab 100644 --- a/lib/mix/lib/mix/app_loader.ex +++ b/lib/mix/lib/mix/app_loader.ex @@ -136,8 +136,12 @@ defmodule Mix.AppLoader do end # We have processed all apps. - defp traverse_apps([], _seen, _deps_children, _builtin_paths, _lib_path) do - [] + defp traverse_apps([], seen, _deps_children, builtin_paths, _lib_path) do + # We want to keep erts in the load path but it doesn't require to be loaded. + case builtin_paths do + %{erts: path} when not is_map_key(seen, :erts) -> [{:erts, path}] + %{} -> [] + end end defp app_children(app) do diff --git a/lib/mix/lib/mix/compilers/elixir.ex b/lib/mix/lib/mix/compilers/elixir.ex index 2bfd19ff2f1..f46836bd63c 100644 --- a/lib/mix/lib/mix/compilers/elixir.ex +++ b/lib/mix/lib/mix/compilers/elixir.ex @@ -1,7 +1,7 @@ defmodule Mix.Compilers.Elixir do @moduledoc false - @manifest_vsn 19 + @manifest_vsn 20 @checkpoint_vsn 2 import Record @@ -46,6 +46,11 @@ defmodule Mix.Compilers.Elixir do {all_modules, all_sources, all_local_exports, old_parents, old_cache_key, old_deps_config} = parse_manifest(manifest, dest) + # Prepend ourselves early because of __mix_recompile__? checks + # and also that, in case of nothing compiled, we already need + # ourselves available in the path. + Code.prepend_path(dest) + # If modules have been added or removed from the Erlang compiler, # we need to recompile all references to old and new modules. stale = @@ -56,7 +61,7 @@ defmodule Mix.Compilers.Elixir do [] end - local_deps = Enum.reject(Mix.Dep.cached(), & &1.scm.fetchable?) + local_deps = Enum.reject(Mix.Dep.cached(), & &1.scm.fetchable?()) # If mix.exs has changed, recompile anything that calls Mix.Project. stale = @@ -393,7 +398,7 @@ defmodule Mix.Compilers.Elixir do Enum.any?(modules, &Map.has_key?(modules_to_recompile, &1)) or Enum.any?(external, &stale_external?(&1, modified, sources_stats)) or (last_mtime > modified and - (missing_beam_file?(dest, modules) or digest != digest_file!(source))), + (missing_beam_file?(dest, modules) or digest_changed?(source, digest))), do: source changed = new_paths ++ changed @@ -423,7 +428,7 @@ defmodule Mix.Compilers.Elixir do defp stale_external?({external, digest}, modified, sources_stats) do case sources_stats do %{^external => {0, 0}} -> digest != nil - %{^external => {mtime, _}} -> mtime > modified and digest != digest_file!(external) + %{^external => {mtime, _}} -> mtime > modified and digest_changed?(external, digest) end end @@ -437,8 +442,11 @@ defmodule Mix.Compilers.Elixir do end) end - defp digest_file!(file) do - file |> File.read!() |> digest_contents() + defp digest_changed?(file, digest) do + case File.read(file) do + {:ok, binary} -> digest != digest_contents(binary) + {:error, _} -> true + end end defp digest_contents(contents) do @@ -446,6 +454,9 @@ defmodule Mix.Compilers.Elixir do 8 -> :crypto.hash(:blake2b, contents) _ -> :crypto.hash(:blake2s, contents) end + rescue + # Blake may not be available on all OpenSSL distribution + _ -> :erlang.md5(contents) end defp set_compiler_opts(opts) do @@ -558,18 +569,18 @@ defmodule Mix.Compilers.Elixir do Enum.any?(enumerable, &Map.has_key?(map, &1)) end - defp stale_local_deps(local_deps, manifest, stale_modules, modified, old_exports) do + defp stale_local_deps(local_deps, manifest, stale_modules, modified, deps_exports) do base = Path.basename(manifest) # The stale modules so far will become both stale_modules and stale_exports, # as any export from a dependency needs to be recompiled. stale_modules = Map.from_keys(stale_modules, true) - for %{opts: opts} <- local_deps, + for %{app: app, opts: opts} <- local_deps, manifest = Path.join([opts[:build], ".mix", base]), Mix.Utils.last_modified(manifest) > modified, - reduce: {stale_modules, stale_modules, old_exports} do - {modules, exports, new_exports} -> + reduce: {stale_modules, stale_modules, deps_exports} do + {modules, exports, deps_exports} -> {manifest_modules, manifest_sources} = read_manifest(manifest) dep_modules = @@ -584,9 +595,11 @@ defmodule Mix.Compilers.Elixir do {dep_modules, _, _} = fixpoint_runtime_modules(manifest_sources, Map.from_keys(dep_modules, true)) + old_exports = Map.get(deps_exports, app, %{}) + # Update exports {exports, new_exports} = - for {module, _} <- dep_modules, reduce: {exports, new_exports} do + for {module, _} <- dep_modules, reduce: {exports, []} do {exports, new_exports} -> export = exports_md5(module, false) @@ -600,17 +613,27 @@ defmodule Mix.Compilers.Elixir do do: exports, else: Map.put(exports, module, true) - # In any case, we always store it as the most update export - # that we have, otherwise we delete it. + # Then we store the new export if any new_exports = if export, - do: Map.put(new_exports, module, export), - else: Map.delete(new_exports, module) + do: [{module, export} | new_exports], + else: new_exports {exports, new_exports} end - {Map.merge(modules, dep_modules), exports, new_exports} + new_exports = Map.new(new_exports) + + removed = + for {module, _} <- old_exports, + not is_map_key(new_exports, module), + do: {module, true}, + into: %{} + + modules = modules |> Map.merge(dep_modules) |> Map.merge(removed) + exports = Map.merge(exports, removed) + deps_exports = Map.put(deps_exports, app, new_exports) + {modules, exports, deps_exports} end end @@ -1081,7 +1104,7 @@ defmodule Mix.Compilers.Elixir do source( source, # We preserve the digest if the file is recompiled but not changed - digest: source(source, :digest) || digest_file!(file), + digest: source(source, :digest) || file |> File.read!() |> digest_contents(), compile_references: compile_references, export_references: export_references, runtime_references: runtime_references, diff --git a/lib/mix/lib/mix/compilers/erlang.ex b/lib/mix/lib/mix/compilers/erlang.ex index d589cedcb7a..7cc2fa58c19 100644 --- a/lib/mix/lib/mix/compilers/erlang.ex +++ b/lib/mix/lib/mix/compilers/erlang.ex @@ -4,13 +4,13 @@ defmodule Mix.Compilers.Erlang do @manifest_vsn 1 @doc """ - Compiles the files in `mappings` with given extensions into - the destination, automatically invoking the callback for each - stale input and output pair (or for all if `force` is `true`) and - removing files that no longer have a source, while keeping the - `manifest` up to date. + Compiles the given `mappings`. - `mappings` should be a list of tuples in the form of `{src, dest}` paths. + `mappings` is a list of `{src, dest}` pairs, where the source + extensions are compiled into the destination extension, + automatically invoking the callback for each stale pair (or for + all if `force` is `true`) and removing files that no longer have + a source, while keeping the `manifest` up to date. ## Options @@ -18,6 +18,9 @@ defmodule Mix.Compilers.Erlang do * `:parallel` - a mapset of files to compile in parallel + * `:preload` - any code that must be preloaded if any pending + entry needs to be compiled + ## Examples For example, a simple compiler for Lisp Flavored Erlang @@ -54,22 +57,25 @@ defmodule Mix.Compilers.Erlang do def compile(manifest, mappings, src_ext, dest_ext, opts, callback) when is_list(opts) do force = opts[:force] - files = + entries = for {src, dest} <- mappings, - target <- extract_targets(src, src_ext, dest, dest_ext, force), + target <- extract_entries(src, src_ext, dest, dest_ext, force), do: target - compile(manifest, files, src_ext, opts, callback) + if preload = entries != [] && opts[:preload] do + preload.() + end + + compile(manifest, entries, src_ext, opts, callback) end @doc """ - Compiles the given `mappings`. + Compiles the given `entries`. - `mappings` should be a list of tuples in the form of `{src, dest}`. + `entries` are a list of `{:ok | :stale, src, dest}` tuples. - A `manifest` file and a `callback` to be invoked for each src/dest pair - must be given. A src/dest pair where destination is `nil` is considered - to be up to date and won't be (re-)compiled. + A `manifest` file and a `callback` to be invoked for each stale + src/dest pair must also be given. ## Options @@ -78,8 +84,8 @@ defmodule Mix.Compilers.Erlang do * `:parallel` - a mapset of files to compile in parallel """ - def compile(manifest, mappings, opts \\ [], callback) do - compile(manifest, mappings, :erl, opts, callback) + def compile(manifest, entries, opts \\ [], callback) do + compile(manifest, entries, :erl, opts, callback) end defp compile(manifest, mappings, ext, opts, callback) do @@ -102,7 +108,7 @@ defmodule Mix.Compilers.Erlang do # Clear stale and removed files from manifest entries = Enum.reject(entries, fn {dest, _warnings} -> - dest in removed || Enum.any?(stale, fn {_, stale_dest} -> dest == stale_dest end) + dest in removed || List.keymember?(stale, dest, 1) end) if Keyword.get(opts, :all_warnings, true), do: show_warnings(entries) @@ -156,9 +162,8 @@ defmodule Mix.Compilers.Erlang do end end - @doc """ - Ensures the native OTP application is available. - """ + # TODO: Deprecate this in favor of `Mix.ensure_application!/1` in Elixir v1.19. + @doc false def ensure_application!(app, _input) do Mix.ensure_application!(app) {:ok, _} = Application.ensure_all_started(app) @@ -202,7 +207,7 @@ defmodule Mix.Compilers.Erlang do manifest |> read_manifest() |> Enum.map(&elem(&1, 0)) end - defp extract_targets(src_dir, src_ext, dest_dir, dest_ext, force) do + defp extract_entries(src_dir, src_ext, dest_dir, dest_ext, force) do files = Mix.Utils.extract_files(List.wrap(src_dir), List.wrap(src_ext)) for file <- files do diff --git a/lib/mix/lib/mix/dep.ex b/lib/mix/lib/mix/dep.ex index 3ab6c32a5af..7efb288a424 100644 --- a/lib/mix/lib/mix/dep.ex +++ b/lib/mix/lib/mix/dep.ex @@ -114,12 +114,12 @@ defmodule Mix.Dep do write_cached_deps(top, {env, target}, load_and_cache(config, top, bottom, env, target)) _ -> - converge(env: env, target: target) + converge_and_load(env: env, target: target) end end defp load_and_cache(_config, top, top, env, target) do - converge(env: env, target: target) + converge_and_load(env: env, target: target) end defp load_and_cache(config, _top, bottom, _env, _target) do @@ -152,6 +152,20 @@ defmodule Mix.Dep do end) end + defp converge_and_load(opts) do + for %{app: app, opts: opts} = dep <- Mix.Dep.Converger.converge(opts) do + case Keyword.pop(opts, :app_properties) do + {nil, _opts} -> + dep + + {app_properties, opts} -> + # We don't raise because child dependencies may be missing if manually cleaned + :application.load({:application, app, app_properties}) + %{dep | opts: opts} + end + end + end + defp read_cached_deps(project, env_target) do case Mix.State.read_cache({:cached_deps, project}) do {^env_target, deps} -> deps @@ -164,6 +178,13 @@ defmodule Mix.Dep do deps end + @doc """ + Although private API, this is kept to avoid breaking code on patch releases. + """ + def load_on_environment(opts) do + Mix.Dep.Converger.converge(opts) + end + @doc """ Clears loaded dependencies from the cache for the current environment. """ @@ -174,25 +195,6 @@ defmodule Mix.Dep do end end - @doc """ - Returns loaded dependencies recursively on the given environment. - - If no environment is passed, dependencies are loaded across all - environments. The result is not cached. - - ## Exceptions - - This function raises an exception if any of the dependencies - provided in the project are in the wrong format. - """ - def load_on_environment(opts) do - converge(opts) - end - - defp converge(opts) do - Mix.Dep.Converger.converge(nil, nil, opts, &{&1, &2, &3}) |> elem(0) - end - @doc """ Filters the given dependencies by name. diff --git a/lib/mix/lib/mix/dep/converger.ex b/lib/mix/lib/mix/dep/converger.ex index 6d1431df23c..7a55039422d 100644 --- a/lib/mix/lib/mix/dep/converger.ex +++ b/lib/mix/lib/mix/dep/converger.ex @@ -60,6 +60,18 @@ defmodule Mix.Dep.Converger do ) end + @doc """ + Converge without lock and accumulator updates. + + Note the dependencies returned from converge are not yet loaded. + The relevant app keys are found under `dep.opts[:app_properties]`. + + See `Mix.Dep.Loader.children/1` for options. + """ + def converge(opts \\ []) do + converge(nil, nil, opts, &{&1, &2, &3}) |> elem(0) + end + @doc """ Converges all dependencies from the current project, including nested dependencies. @@ -68,11 +80,14 @@ defmodule Mix.Dep.Converger do must return an updated dependency in case some processing is done. + Note the dependencies returned from converge are not yet loaded. + The relevant app keys are found under `dep.opts[:app_properties]`. + See `Mix.Dep.Loader.children/1` for options. """ def converge(acc, lock, opts, callback) do {deps, acc, lock} = all(acc, lock, opts, callback) - if remote = Mix.RemoteConverger.get(), do: remote.post_converge + if remote = Mix.RemoteConverger.get(), do: remote.post_converge() {topological_sort(deps), acc, lock} end @@ -205,21 +220,20 @@ defmodule Mix.Dep.Converger do all(t, [dep | acc], upper, breadths, optional, rest, lock, state) :nomatch -> - {%{app: app, deps: deps, opts: opts} = dep, app_properties, rest, lock} = + {%{app: app, deps: deps, opts: opts} = dep, rest, lock} = case state.cache.(dep) do {:loaded, cached_dep} -> - {cached_dep, nil, rest, lock} + {cached_dep, rest, lock} {:unloaded, dep, children} -> {dep, rest, lock} = state.callback.(put_lock(dep, lock), rest, lock) - Mix.Dep.Loader.with_system_env(dep, fn -> - # After we invoke the callback (which may actually check out the - # dependency), we load the dependency including its latest info - # and children information. - {dep, app_properties} = Mix.Dep.Loader.load(dep, children, state.locked?) - {dep, app_properties, rest, lock} - end) + dep = + Mix.Dep.Loader.with_system_env(dep, fn -> + Mix.Dep.Loader.load(dep, children, state.locked?) + end) + + {dep, rest, lock} end # Something that we previously ruled out as an optional dependency is @@ -234,24 +248,7 @@ defmodule Mix.Dep.Converger do split_non_fulfilled_optional(deps, Enum.map(acc, & &1.app), opts[:from_umbrella]) new_breadths = Enum.map(deps, & &1.app) ++ breadths - res = all(deps, acc, breadths, new_breadths, discarded ++ optional, rest, lock, state) - - # After we traverse all of our children, we can load ourselves. - # This is important in case of included application. - if app_properties do - case :application.load({:application, app, app_properties}) do - :ok -> - :ok - - {:error, {:already_loaded, _}} -> - :ok - - {:error, error} -> - Mix.raise("Could not start application #{inspect(app)}: #{inspect(error)}") - end - end - - res + all(deps, acc, breadths, new_breadths, discarded ++ optional, rest, lock, state) end end diff --git a/lib/mix/lib/mix/dep/loader.ex b/lib/mix/lib/mix/dep/loader.ex index 92679ffc810..49d7f6c1ef8 100644 --- a/lib/mix/lib/mix/dep/loader.ex +++ b/lib/mix/lib/mix/dep/loader.ex @@ -383,21 +383,21 @@ defmodule Mix.Dep.Loader do cond do not ok?(dep) -> - {dep, nil} + dep recently_fetched?(dep) -> - {%{dep | status: :compile}, nil} + %{dep | status: :compile} opts_app == false -> - {dep, nil} + dep true -> path = if is_binary(opts_app), do: opts_app, else: "ebin/#{app}.app" path = Path.expand(path, opts[:build]) case app_status(path, app, req) do - {:ok, vsn, app} -> {%{dep | status: {:ok, vsn}}, app} - status -> {%{dep | status: status}, nil} + {:ok, vsn, app} -> %{dep | status: {:ok, vsn}, opts: [app_properties: app] ++ opts} + status -> %{dep | status: status} end end end diff --git a/lib/mix/lib/mix/dep/umbrella.ex b/lib/mix/lib/mix/dep/umbrella.ex index f319b972158..023f627383f 100644 --- a/lib/mix/lib/mix/dep/umbrella.ex +++ b/lib/mix/lib/mix/dep/umbrella.ex @@ -59,7 +59,7 @@ defmodule Mix.Dep.Umbrella do apps = Enum.map(deps, & &1.app) Enum.map(deps, fn umbrella_dep -> - {umbrella_dep, _} = Mix.Dep.Loader.load(umbrella_dep, nil, false) + umbrella_dep = Mix.Dep.Loader.load(umbrella_dep, nil, false) deps = Enum.filter(umbrella_dep.deps, fn dep -> diff --git a/lib/mix/lib/mix/release.ex b/lib/mix/lib/mix/release.ex index 0d5cb900cd6..316a60a81bf 100644 --- a/lib/mix/lib/mix/release.ex +++ b/lib/mix/lib/mix/release.ex @@ -67,7 +67,7 @@ defmodule Mix.Release do @default_apps [kernel: :permanent, stdlib: :permanent, elixir: :permanent, sasl: :permanent] @safe_modes [:permanent, :temporary, :transient] @unsafe_modes [:load, :none] - @significant_chunks ~w(Atom AtU8 Attr Code StrT ImpT ExpT FunT LitT Line)c + @additional_chunks ~w(Attr)c @copy_app_dirs ["priv"] @doc false @@ -285,7 +285,8 @@ defmodule Mix.Release do |> Enum.map(&{&1, new_mode}) seen = put_in(seen[app][:mode], new_mode) - load_apps(apps, deps_apps, seen, otp_root, [], overrides) + optional = Keyword.get(properties, :optional_applications, []) + load_apps(apps, deps_apps, seen, otp_root, optional, overrides) true -> seen @@ -878,7 +879,7 @@ defmodule Mix.Release do @spec strip_beam(binary(), keyword()) :: {:ok, binary()} | {:error, :beam_lib, term()} def strip_beam(binary, options \\ []) when is_list(options) do chunks_to_keep = options[:keep] |> List.wrap() |> Enum.map(&String.to_charlist/1) - all_chunks = Enum.uniq(@significant_chunks ++ chunks_to_keep) + all_chunks = Enum.uniq(@additional_chunks ++ :beam_lib.significant_chunks() ++ chunks_to_keep) compress? = Keyword.get(options, :compress, false) case :beam_lib.chunks(binary, all_chunks, [:allow_missing_chunks]) do diff --git a/lib/mix/lib/mix/task.ex b/lib/mix/lib/mix/task.ex index 19c892a2bd8..029c40b13ba 100644 --- a/lib/mix/lib/mix/task.ex +++ b/lib/mix/lib/mix/task.ex @@ -58,14 +58,22 @@ defmodule Mix.Task do @requirements ["app.config"] - Tasks typically depend on the `"app.config"` task, when they - need to access code from the current project with all apps - already configured, or the "app.start" task, when they also - need those apps to be already started: - - @requirements ["app.start"] - - You can also run tasks directly with `run/2`. + A task will typically depend on one of the following tasks: + + * "loadpaths" - this ensures dependencies are available + and compiled. If you are publishing a task as part of + a library to be used by others, and your task does not + need to interact with the user code in any way, this is + the recommended requirement + + * "app.config" - additionally compiles and loads the runtime + configuration for the current project. If you are creating + a task to be used within your application or as part of a + library, which must invoke or interact with the user code, + this is the minimum recommended requirement + + * "app.start" - additionally starts the supervision tree of + the current project and its dependencies Finally, set `@recursive true` if you want the task to run on each umbrella child in an umbrella project. diff --git a/lib/mix/lib/mix/tasks/archive.build.ex b/lib/mix/lib/mix/tasks/archive.build.ex index 8e5451a74f9..852264d735d 100644 --- a/lib/mix/lib/mix/tasks/archive.build.ex +++ b/lib/mix/lib/mix/tasks/archive.build.ex @@ -61,7 +61,7 @@ defmodule Mix.Tasks.Archive.Build do project = Mix.Project.get() if project && Keyword.get(opts, :compile, true) do - Mix.Task.run(:compile, args) + Mix.Task.run(:compile, ["--no-protocol-consolidation" | args]) end source = diff --git a/lib/mix/lib/mix/tasks/compile.all.ex b/lib/mix/lib/mix/tasks/compile.all.ex index d48e17dc348..8f29a285612 100644 --- a/lib/mix/lib/mix/tasks/compile.all.ex +++ b/lib/mix/lib/mix/tasks/compile.all.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Compile.All do {loaded_paths, loaded_modules} = Mix.AppLoader.load_apps(apps, deps, config, {[], []}, fn {app, path}, {paths, mods} -> paths = if path, do: [path | paths], else: paths - mods = if app_cache, do: [{app, Application.spec(app, :modules)} | mods], else: mods + mods = if app_cache, do: [{app, Application.spec(app, :modules) || []} | mods], else: mods {paths, mods} end) @@ -43,7 +43,11 @@ defmodule Mix.Tasks.Compile.All do Code.delete_paths(current_paths -- loaded_paths) end - Code.prepend_paths(loaded_paths -- current_paths, cache: true) + # Add the current compilation path. compile.elixir and compile.erlang + # will also add this path, but only if they run, so we always add it + # here too. Furthermore, we don't cache it as we may still write to it. + compile_path = to_charlist(Mix.Project.compile_path()) + Code.prepend_paths([compile_path | loaded_paths -- current_paths], cache: true) result = if "--no-compile" in args do @@ -64,12 +68,6 @@ defmodule Mix.Tasks.Compile.All do Mix.AppLoader.write_cache(app_cache, Map.new(loaded_modules)) end - # Add the current compilation path. compile.elixir and compile.erlang - # will also add this path, but only if they run, so we always add it - # here too. Furthermore, we don't cache it as we may still write to it. - compile_path = to_charlist(Mix.Project.compile_path()) - _ = Code.prepend_path(compile_path) - unless "--no-app-loading" in args do app = config[:app] @@ -125,7 +123,7 @@ defmodule Mix.Tasks.Compile.All do Enum.reduce(Mix.ProjectStack.pop_after_compiler(compiler), result, & &1.(&2)) end - defp project_apps(config) do + def project_apps(config) do project = Mix.Project.get!() properties = diff --git a/lib/mix/lib/mix/tasks/compile.ex b/lib/mix/lib/mix/tasks/compile.ex index 73cb788da98..3f3484c3abe 100644 --- a/lib/mix/lib/mix/tasks/compile.ex +++ b/lib/mix/lib/mix/tasks/compile.ex @@ -146,10 +146,9 @@ defmodule Mix.Tasks.Compile do config = Mix.Project.config() # If we are in an umbrella project, now load paths from all children. - if Mix.Project.umbrella?(config) do + if apps_paths = Mix.Project.apps_paths(config) do loaded_paths = - Mix.Project.apps_paths(config) - |> Map.keys() + (Mix.Tasks.Compile.All.project_apps(config) ++ Map.keys(apps_paths)) |> Mix.AppLoader.load_apps(Mix.Dep.cached(), config, [], fn {_app, path}, acc -> if path, do: [path | acc], else: acc end) diff --git a/lib/mix/lib/mix/tasks/compile.leex.ex b/lib/mix/lib/mix/tasks/compile.leex.ex index 695423bbf9d..cfd541067ac 100644 --- a/lib/mix/lib/mix/tasks/compile.leex.ex +++ b/lib/mix/lib/mix/tasks/compile.leex.ex @@ -53,15 +53,19 @@ defmodule Mix.Tasks.Compile.Leex do Mix.raise(":leex_options should be a list of options, got: #{inspect(options)}") end - opts = [parallel: true] ++ opts + opts = [parallel: true, preload: &preload/0] ++ opts Erlang.compile(manifest(), mappings, :xrl, :erl, opts, fn input, output -> - Erlang.ensure_application!(:parsetools, input) options = options ++ @forced_opts ++ [scannerfile: Erlang.to_erl_file(output)] :leex.file(Erlang.to_erl_file(input), options) end) end + defp preload do + Mix.ensure_application!(:parsetools) + {:ok, _} = Application.ensure_all_started(:parsetools) + end + @impl true def manifests, do: [manifest()] defp manifest, do: Path.join(Mix.Project.manifest_path(), @manifest) diff --git a/lib/mix/lib/mix/tasks/compile.yecc.ex b/lib/mix/lib/mix/tasks/compile.yecc.ex index 9698df584d9..02c9700c972 100644 --- a/lib/mix/lib/mix/tasks/compile.yecc.ex +++ b/lib/mix/lib/mix/tasks/compile.yecc.ex @@ -53,15 +53,19 @@ defmodule Mix.Tasks.Compile.Yecc do Mix.raise(":yecc_options should be a list of options, got: #{inspect(options)}") end - opts = [parallel: true] ++ opts + opts = [parallel: true, preload: &preload/0] ++ opts Erlang.compile(manifest(), mappings, :yrl, :erl, opts, fn input, output -> - Erlang.ensure_application!(:parsetools, input) options = options ++ @forced_opts ++ [parserfile: Erlang.to_erl_file(output)] :yecc.file(Erlang.to_erl_file(input), options) end) end + defp preload do + Mix.ensure_application!(:parsetools) + {:ok, _} = Application.ensure_all_started(:parsetools) + end + @impl true def manifests, do: [manifest()] defp manifest, do: Path.join(Mix.Project.manifest_path(), @manifest) diff --git a/lib/mix/lib/mix/tasks/deps.clean.ex b/lib/mix/lib/mix/tasks/deps.clean.ex index 2d10e5d0aa3..1fbdb41e743 100644 --- a/lib/mix/lib/mix/tasks/deps.clean.ex +++ b/lib/mix/lib/mix/tasks/deps.clean.ex @@ -41,7 +41,7 @@ defmodule Mix.Tasks.Deps.Clean do value = opts[switch], do: {key, :"#{value}"} - loaded_deps = Mix.Dep.load_on_environment(loaded_opts) + loaded_deps = Mix.Dep.Converger.converge(loaded_opts) apps_to_clean = cond do diff --git a/lib/mix/lib/mix/tasks/deps.compile.ex b/lib/mix/lib/mix/tasks/deps.compile.ex index de1e68d3b6e..84ba8a3c8b8 100644 --- a/lib/mix/lib/mix/tasks/deps.compile.ex +++ b/lib/mix/lib/mix/tasks/deps.compile.ex @@ -57,9 +57,6 @@ defmodule Mix.Tasks.Deps.Compile do case OptionParser.parse(args, switches: @switches) do {opts, [], _} -> - # Because this command may be invoked explicitly with - # deps.compile, we simply try to compile any available - # or local dependency. compile(filter_available_and_local_deps(deps), opts) {opts, tail, _} -> diff --git a/lib/mix/lib/mix/tasks/deps.ex b/lib/mix/lib/mix/tasks/deps.ex index 3a057cb9a5b..ff51f089dcf 100644 --- a/lib/mix/lib/mix/tasks/deps.ex +++ b/lib/mix/lib/mix/tasks/deps.ex @@ -1,7 +1,7 @@ defmodule Mix.Tasks.Deps do use Mix.Task - import Mix.Dep, only: [load_on_environment: 1, format_dep: 1, format_status: 1, check_lock: 1] + import Mix.Dep, only: [format_dep: 1, format_status: 1, check_lock: 1] @shortdoc "Lists dependencies and their status" @@ -167,7 +167,7 @@ defmodule Mix.Tasks.Deps do shell = Mix.shell() - load_on_environment(loaded_opts) + Mix.Dep.Converger.converge(loaded_opts) |> Enum.sort_by(& &1.app) |> Enum.each(fn dep -> %Mix.Dep{scm: scm, manager: manager} = dep diff --git a/lib/mix/lib/mix/tasks/deps.loadpaths.ex b/lib/mix/lib/mix/tasks/deps.loadpaths.ex index 9b4679a33b8..9425075339b 100644 --- a/lib/mix/lib/mix/tasks/deps.loadpaths.ex +++ b/lib/mix/lib/mix/tasks/deps.loadpaths.ex @@ -17,7 +17,6 @@ defmodule Mix.Tasks.Deps.Loadpaths do * `--no-archives-check` - does not check archives * `--no-compile` - does not compile even if files require compilation * `--no-deps-check` - does not check or compile deps, only load available ones - * `--no-deps-loading` - does not add deps loadpaths to the code path * `--no-elixir-version-check` - does not check Elixir version * `--no-optional-deps` - does not compile or load optional deps diff --git a/lib/mix/lib/mix/tasks/deps.tree.ex b/lib/mix/lib/mix/tasks/deps.tree.ex index 368fca84ffb..de3d40dea7f 100644 --- a/lib/mix/lib/mix/tasks/deps.tree.ex +++ b/lib/mix/lib/mix/tasks/deps.tree.ex @@ -44,7 +44,7 @@ defmodule Mix.Tasks.Deps.Tree do value = opts[switch], do: {key, :"#{value}"} - deps = Mix.Dep.load_on_environment(deps_opts) + deps = Mix.Dep.Converger.converge(deps_opts) root = case args do diff --git a/lib/mix/lib/mix/tasks/deps.unlock.ex b/lib/mix/lib/mix/tasks/deps.unlock.ex index ba4f490cba0..c0a601269b8 100644 --- a/lib/mix/lib/mix/tasks/deps.unlock.ex +++ b/lib/mix/lib/mix/tasks/deps.unlock.ex @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Deps.Unlock do end defp unused_apps(lock) do - apps = Mix.Dep.load_on_environment([]) |> Enum.map(& &1.app) + apps = Mix.Dep.Converger.converge([]) |> Enum.map(& &1.app) lock |> Map.drop(apps) diff --git a/lib/mix/lib/mix/tasks/format.ex b/lib/mix/lib/mix/tasks/format.ex index 097aba5e0fb..d4ee5700394 100644 --- a/lib/mix/lib/mix/tasks/format.ex +++ b/lib/mix/lib/mix/tasks/format.ex @@ -233,6 +233,8 @@ defmodule Mix.Tasks.Format do {formatter_opts_and_subs, _sources} = eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, [dot_formatter]) + formatter_opts_and_subs = load_plugins(formatter_opts_and_subs) + args |> expand_args(cwd, dot_formatter, formatter_opts_and_subs, opts) |> Task.async_stream(&format_file(&1, opts), ordered: false, timeout: :infinity) @@ -240,6 +242,55 @@ defmodule Mix.Tasks.Format do |> check!(opts) end + defp load_plugins({formatter_opts, subs}) do + plugins = Keyword.get(formatter_opts, :plugins, []) + + if not is_list(plugins) do + Mix.raise("Expected :plugins to return a list of directories, got: #{inspect(plugins)}") + end + + if plugins != [] do + Mix.Task.run("loadpaths", []) + end + + if not Enum.all?(plugins, &Code.ensure_loaded?/1) do + Mix.Task.run("compile", []) + end + + for plugin <- plugins do + cond do + not Code.ensure_loaded?(plugin) -> + Mix.raise("Formatter plugin #{inspect(plugin)} cannot be found") + + not function_exported?(plugin, :features, 1) -> + Mix.raise("Formatter plugin #{inspect(plugin)} does not define features/1") + + true -> + :ok + end + end + + sigils = + for plugin <- plugins, + sigil <- find_sigils_from_plugins(plugin, formatter_opts), + do: {sigil, plugin} + + sigils = + sigils + |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) + |> Enum.map(fn {sigil, plugins} -> + {sigil, + fn input, opts -> + Enum.reduce(plugins, input, fn plugin, input -> + plugin.format(input, opts ++ formatter_opts) + end) + end} + end) + + {Keyword.put(formatter_opts, :sigils, sigils), + Enum.map(subs, fn {path, opts} -> {path, load_plugins(opts)} end)} + end + @doc """ Returns a formatter function and the formatter options to be used for the given file. @@ -256,6 +307,8 @@ defmodule Mix.Tasks.Format do {formatter_opts_and_subs, _sources} = eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, [dot_formatter]) + formatter_opts_and_subs = load_plugins(formatter_opts_and_subs) + find_formatter_and_opts_for_file(Path.expand(file, cwd), formatter_opts_and_subs) end @@ -289,7 +342,6 @@ defmodule Mix.Tasks.Format do defp eval_deps_and_subdirectories(cwd, dot_formatter, formatter_opts, sources) do deps = Keyword.get(formatter_opts, :import_deps, []) subs = Keyword.get(formatter_opts, :subdirectories, []) - plugins = Keyword.get(formatter_opts, :plugins, []) if not is_list(deps) do Mix.raise("Expected :import_deps to return a list of dependencies, got: #{inspect(deps)}") @@ -299,53 +351,6 @@ defmodule Mix.Tasks.Format do Mix.raise("Expected :subdirectories to return a list of directories, got: #{inspect(subs)}") end - if not is_list(plugins) do - Mix.raise("Expected :plugins to return a list of modules, got: #{inspect(plugins)}") - end - - if plugins != [] do - Mix.Task.run("loadpaths", []) - end - - if not Enum.all?(plugins, &Code.ensure_loaded?/1) do - Mix.Task.run("compile", []) - end - - for plugin <- plugins do - cond do - not Code.ensure_loaded?(plugin) -> - Mix.raise("Formatter plugin #{inspect(plugin)} cannot be found") - - not function_exported?(plugin, :features, 1) -> - Mix.raise("Formatter plugin #{inspect(plugin)} does not define features/1") - - true -> - :ok - end - end - - sigils = - for plugin <- plugins, - sigil <- find_sigils_from_plugins(plugin, formatter_opts), - do: {sigil, plugin} - - sigils = - sigils - |> Enum.group_by(&elem(&1, 0), &elem(&1, 1)) - |> Enum.map(fn {sigil, plugins} -> - {sigil, - fn input, opts -> - Enum.reduce(plugins, input, fn plugin, input -> - plugin.format(input, opts ++ formatter_opts) - end) - end} - end) - - formatter_opts = - formatter_opts - |> Keyword.put(:plugins, plugins) - |> Keyword.put(:sigils, sigils) - if deps == [] and subs == [] do {{formatter_opts, []}, sources} else @@ -448,7 +453,8 @@ defmodule Mix.Tasks.Format do _ -> Mix.raise( "Unknown dependency #{inspect(dep)} given to :import_deps in the formatter configuration. " <> - "Make sure the dependency is listed in your mix.exs and you have run \"mix deps.get\"" + "Make sure the dependency is listed in your mix.exs for environment #{inspect(Mix.env())} " <> + "and you have run \"mix deps.get\"" ) end end @@ -578,8 +584,15 @@ defmodule Mix.Tasks.Format do defp recur_formatter_opts_for_file(file, {formatter_opts, subs}) do Enum.find_value(subs, formatter_opts, fn {sub, formatter_opts_and_subs} -> - if String.starts_with?(file, sub) do - recur_formatter_opts_for_file(file, formatter_opts_and_subs) + size = byte_size(sub) + + case file do + <> + when prefix == sub and dir_separator in [?\\, ?/] -> + recur_formatter_opts_for_file(file, formatter_opts_and_subs) + + _ -> + nil end end) end diff --git a/lib/mix/lib/mix/tasks/loadpaths.ex b/lib/mix/lib/mix/tasks/loadpaths.ex index 9a58332829f..0ea7fc3a6ed 100644 --- a/lib/mix/lib/mix/tasks/loadpaths.ex +++ b/lib/mix/lib/mix/tasks/loadpaths.ex @@ -20,7 +20,6 @@ defmodule Mix.Tasks.Loadpaths do * `--no-archives-check` - does not check archives * `--no-compile` - does not compile dependencies, only check and load them * `--no-deps-check` - does not check dependencies, only load available ones - * `--no-deps-loading` - does not add deps loadpaths to the code path * `--no-elixir-version-check` - does not check Elixir version * `--no-optional-deps` - does not compile or load optional deps diff --git a/lib/mix/lib/mix/tasks/release.ex b/lib/mix/lib/mix/tasks/release.ex index d7ec3fe253f..388bea48113 100644 --- a/lib/mix/lib/mix/tasks/release.ex +++ b/lib/mix/lib/mix/tasks/release.ex @@ -1425,7 +1425,16 @@ defmodule Mix.Tasks.Release do defp cli_for(:windows, release) do {"env.bat", &env_bat_template(release: &1), - [{"#{release.name}.bat", cli_bat_template(release: release)}]} + [{"#{release.name}.bat", cli_bat_template(release: release) |> maybe_replace_werl()}]} + end + + defp maybe_replace_werl(contents) do + # TODO: Remove me when we require Erlang/OTP 26+ + if :erlang.system_info(:otp_release) >= ~c"26" do + String.replace(contents, "--werl", "") + else + contents + end end defp elixir_cli_for(:unix, release) do diff --git a/lib/mix/lib/mix/tasks/xref.ex b/lib/mix/lib/mix/tasks/xref.ex index 8403e79679d..bf55ebb47df 100644 --- a/lib/mix/lib/mix/tasks/xref.ex +++ b/lib/mix/lib/mix/tasks/xref.ex @@ -38,8 +38,9 @@ defmodule Mix.Tasks.Xref do $ mix xref trace lib/my_app/router.ex --label compile If you have an umbrella application, we also recommend using the - `--include-siblings` flag to see the dependencies on other - umbrella applications. + `--include-siblings` flag to see the dependencies from sibling + applications. The `trace` command is not currently supported at the + umbrella root. ### Example @@ -222,6 +223,12 @@ defmodule Mix.Tasks.Xref do lib/a.ex └── lib/b.ex (compile) + If you have an umbrella application, we also recommend using the + `--include-siblings` flag to see the dependencies from sibling + applications. When invoked at the umbrella root, the `graph` + command will list all files from all umbrella children, without + any namespacing. + ### Dependency types Elixir tracks three types of dependencies between modules: compile, @@ -289,12 +296,6 @@ defmodule Mix.Tasks.Xref do @impl true def run(args) do - if Mix.Project.umbrella?() do - Mix.raise( - "mix xref is not supported in the umbrella root. Please run it inside the umbrella applications instead" - ) - end - Mix.Task.run("compile", args) Mix.Task.reenable("xref") @@ -302,9 +303,11 @@ defmodule Mix.Tasks.Xref do case args do ["callers", module] -> + no_umbrella!("callers") handle_callers(module, opts) ["trace", file] -> + no_umbrella!("trace") handle_trace(file, opts) ["graph"] -> @@ -327,6 +330,14 @@ defmodule Mix.Tasks.Xref do end end + defp no_umbrella!(task) do + if Mix.Project.umbrella?() do + Mix.raise( + "mix xref #{task} is not supported in the umbrella root. Please run it inside the umbrella applications instead" + ) + end + end + @doc """ Returns a list of information of all the runtime function calls in the project. @@ -1058,12 +1069,18 @@ defmodule Mix.Tasks.Xref do defp manifests(opts) do siblings = - if opts[:include_siblings] do - for %{scm: Mix.SCM.Path, opts: opts} <- Mix.Dep.cached(), - opts[:in_umbrella], - do: Path.join([opts[:build], ".mix", @manifest]) - else - [] + cond do + Mix.Project.umbrella?() -> + for %{opts: opts} <- Mix.Dep.Umbrella.cached(), + do: Path.join([opts[:build], ".mix", @manifest]) + + opts[:include_siblings] -> + for %{scm: Mix.SCM.Path, opts: opts} <- Mix.Dep.cached(), + opts[:in_umbrella], + do: Path.join([opts[:build], ".mix", @manifest]) + + true -> + [] end [Path.join(Mix.Project.manifest_path(), @manifest) | siblings] diff --git a/lib/mix/test/fixtures/umbrella_dep/deps/umbrella/mix.exs b/lib/mix/test/fixtures/umbrella_dep/deps/umbrella/mix.exs index b8598c013be..705532e0576 100644 --- a/lib/mix/test/fixtures/umbrella_dep/deps/umbrella/mix.exs +++ b/lib/mix/test/fixtures/umbrella_dep/deps/umbrella/mix.exs @@ -4,4 +4,8 @@ defmodule Umbrella.MixProject do def project do [apps_path: "apps"] end + + def application do + [extra_applications: [:runtime_tools]] + end end diff --git a/lib/mix/test/mix/dep_test.exs b/lib/mix/test/mix/dep_test.exs index 9bf45962134..5955e1b054c 100644 --- a/lib/mix/test/mix/dep_test.exs +++ b/lib/mix/test/mix/dep_test.exs @@ -36,7 +36,7 @@ defmodule Mix.DepTest do defp assert_wrong_dependency(deps) do with_deps(deps, fn -> assert_raise Mix.Error, ~r"Dependency specified in the wrong format", fn -> - Mix.Dep.load_on_environment([]) + Mix.Dep.Converger.converge([]) end end) end @@ -59,7 +59,7 @@ defmodule Mix.DepTest do in_fixture("deps_status", fn -> Mix.Project.push(DepsApp) - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert length(deps) == 6 assert Enum.find(deps, &match?(%Mix.Dep{app: :ok, status: {:ok, _}}, &1)) assert Enum.find(deps, &match?(%Mix.Dep{app: :invalidvsn, status: {:invalidvsn, :ok}}, &1)) @@ -102,7 +102,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert Enum.find(deps, &match?(%Mix.Dep{app: :ok, status: {:ok, _}}, &1)) end) end) @@ -115,7 +115,7 @@ defmodule Mix.DepTest do in_fixture("deps_status", fn -> send(self(), {:mix_shell_input, :yes?, false}) msg = "Could not find an SCM for dependency :ok from Mix.DepTest.ProcessDepsApp" - assert_raise Mix.Error, msg, fn -> Mix.Dep.load_on_environment([]) end + assert_raise Mix.Error, msg, fn -> Mix.Dep.Converger.converge([]) end end) end) end @@ -139,7 +139,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> assert_raise Mix.Error, ~r"Invalid requirement", fn -> - Mix.Dep.load_on_environment([]) + Mix.Dep.Converger.converge([]) end end) end) @@ -150,7 +150,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo] end) end) end @@ -172,7 +172,7 @@ defmodule Mix.DepTest do end """) - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:deps_repo] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:deps_repo] end) end) end @@ -185,7 +185,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo] end) end) end @@ -200,7 +200,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - [dep1, dep2] = Mix.Dep.load_on_environment([]) + [dep1, dep2] = Mix.Dep.Converger.converge([]) assert dep1.manager == nil assert dep2.manager == :rebar3 end) @@ -252,7 +252,7 @@ defmodule Mix.DepTest do end """) - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:git_repo, :deps_repo] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:git_repo, :deps_repo] end) end) end @@ -362,7 +362,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_tmp("load dependency with env vars", fn -> - Mix.Dep.load_on_environment([]) + Mix.Dep.Converger.converge([]) assert {:ok, "contents dep test"} = File.read(file_path) end) end) @@ -378,7 +378,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - [git_repo, _] = Mix.Dep.load_on_environment([]) + [git_repo, _] = Mix.Dep.Converger.converge([]) %{app: :git_repo, status: {:overridden, _}} = git_repo end) end) @@ -549,7 +549,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> # Both orders below are valid after topological sort - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) in [ + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) in [ [:git_repo, :abc_repo, :deps_repo], [:abc_repo, :git_repo, :deps_repo] ] @@ -604,7 +604,7 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> # Both orders below are valid after topological sort - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) in [ + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) in [ [:git_repo, :abc_repo, :deps_repo], [:abc_repo, :git_repo, :deps_repo] ] @@ -639,13 +639,13 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - deps = Mix.Dep.load_on_environment(env: :other_env) + deps = Mix.Dep.Converger.converge(env: :other_env) assert length(deps) == 2 - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert length(deps) == 2 - assert [dep] = Mix.Dep.load_on_environment(env: :prod) + assert [dep] = Mix.Dep.Converger.converge(env: :prod) assert dep.app == :foo end) end) @@ -676,10 +676,10 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:deps_repo] = Enum.map(loaded, & &1.app) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:deps_repo] = Enum.map(loaded, & &1.app) end) end) @@ -694,15 +694,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :dev) + loaded = Mix.Dep.Converger.converge(env: :dev) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) @@ -727,15 +727,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :dev) + loaded = Mix.Dep.Converger.converge(env: :dev) assert [:deps_repo] = Enum.map(loaded, & &1.app) assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) end) @@ -751,18 +751,18 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :dev) + loaded = Mix.Dep.Converger.converge(env: :dev) assert [] = Enum.map(loaded, & &1.app) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:git_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :prod) + loaded = Mix.Dep.Converger.converge(env: :prod) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) end) @@ -778,15 +778,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :dev) + loaded = Mix.Dep.Converger.converge(env: :dev) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedonly: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:git_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _] = Enum.map(loaded, & &1.status) @@ -811,19 +811,19 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :dev) + loaded = Mix.Dep.Converger.converge(env: :dev) assert [:deps_repo] = Enum.map(loaded, & &1.app) assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :test) + loaded = Mix.Dep.Converger.converge(env: :test) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(env: :prod) + loaded = Mix.Dep.Converger.converge(env: :prod) assert [] = Enum.map(loaded, & &1.app) end) end) @@ -848,7 +848,7 @@ defmodule Mix.DepTest do """) Mix.State.clear_cache() - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) Enum.map(loaded, &{&1.app, &1.opts[:only]}) end) end) @@ -933,13 +933,13 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - deps = Mix.Dep.load_on_environment(target: :rpi3) + deps = Mix.Dep.Converger.converge(target: :rpi3) assert length(deps) == 2 - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert length(deps) == 2 - assert [dep] = Mix.Dep.load_on_environment(target: :host) + assert [dep] = Mix.Dep.Converger.converge(target: :host) assert dep.app == :foo end) end) @@ -973,15 +973,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :host) + loaded = Mix.Dep.Converger.converge(target: :host) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :rpi3) + loaded = Mix.Dep.Converger.converge(target: :rpi3) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) @@ -1006,15 +1006,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :host) + loaded = Mix.Dep.Converger.converge(target: :host) assert [:deps_repo] = Enum.map(loaded, & &1.app) assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :rpi3) + loaded = Mix.Dep.Converger.converge(target: :rpi3) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) end) @@ -1030,18 +1030,18 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :host) + loaded = Mix.Dep.Converger.converge(target: :host) assert [] = Enum.map(loaded, & &1.app) - loaded = Mix.Dep.load_on_environment(target: :bbb) + loaded = Mix.Dep.Converger.converge(target: :bbb) assert [:git_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :rpi3) + loaded = Mix.Dep.Converger.converge(target: :rpi3) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) end) @@ -1057,15 +1057,15 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :host) + loaded = Mix.Dep.Converger.converge(target: :host) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [divergedtargets: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :rpi3) + loaded = Mix.Dep.Converger.converge(target: :rpi3) assert [:git_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _] = Enum.map(loaded, & &1.status) @@ -1090,19 +1090,19 @@ defmodule Mix.DepTest do with_deps(deps, fn -> in_fixture("deps_status", fn -> - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :host) + loaded = Mix.Dep.Converger.converge(target: :host) assert [:deps_repo] = Enum.map(loaded, & &1.app) assert [noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :bbb) + loaded = Mix.Dep.Converger.converge(target: :bbb) assert [:git_repo, :deps_repo] = Enum.map(loaded, & &1.app) assert [unavailable: _, noappfile: {_, _}] = Enum.map(loaded, & &1.status) - loaded = Mix.Dep.load_on_environment(target: :rpi3) + loaded = Mix.Dep.Converger.converge(target: :rpi3) assert [] = Enum.map(loaded, & &1.app) end) end) @@ -1127,7 +1127,7 @@ defmodule Mix.DepTest do """) Mix.State.clear_cache() - loaded = Mix.Dep.load_on_environment([]) + loaded = Mix.Dep.Converger.converge([]) Enum.map(loaded, &{&1.app, &1.opts[:targets]}) end) end) diff --git a/lib/mix/test/mix/rebar_test.exs b/lib/mix/test/mix/rebar_test.exs index d26443d7ced..a5154b87a83 100644 --- a/lib/mix/test/mix/rebar_test.exs +++ b/lib/mix/test/mix/rebar_test.exs @@ -149,14 +149,14 @@ defmodule Mix.RebarTest do describe "integration with Mix" do test "inherits Rebar manager" do Mix.Project.push(RebarAsDep) - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert Enum.all?(deps, &(&1.manager == :rebar3)) end test "parses Rebar dependencies from rebar.config" do Mix.Project.push(RebarAsDep) - deps = Mix.Dep.load_on_environment([]) + deps = Mix.Dep.Converger.converge([]) assert Enum.all?(deps, &(&1.manager == :rebar3)) assert Enum.find(deps, &(&1.app == :rebar_dep)) @@ -175,7 +175,7 @@ defmodule Mix.RebarTest do Mix.Tasks.Deps.Get.run([]) - assert Mix.Dep.load_on_environment([]) |> Enum.map(& &1.app) == + assert Mix.Dep.Converger.converge([]) |> Enum.map(& &1.app) == [:git_repo, :git_rebar, :rebar_override] end) after @@ -199,7 +199,7 @@ defmodule Mix.RebarTest do assert :rebar_dep.any_function() == :ok load_paths = - Mix.Dep.load_on_environment([]) + Mix.Dep.Converger.converge([]) |> Enum.map(&Mix.Dep.load_paths(&1)) |> Enum.concat() diff --git a/lib/mix/test/mix/release_test.exs b/lib/mix/test/mix/release_test.exs index c2ac0eea3ae..958268dfb66 100644 --- a/lib/mix/test/mix/release_test.exs +++ b/lib/mix/test/mix/release_test.exs @@ -337,17 +337,14 @@ defmodule Mix.ReleaseTest do test "configures other applications in cascade", context do in_tmp(context.test, fn -> - app = + write_app!( + "my_sample_mode/ebin/my_sample_mode.app", {:application, :my_sample_mode, applications: [:kernel, :stdlib, :elixir, :runtime_tools, :compiler], description: ~c"my_sample_mode", modules: [], vsn: ~c"1.0.0"} - - File.mkdir_p!("my_sample_mode/ebin") - Code.prepend_path("my_sample_mode/ebin") - format = :io_lib.format("%% coding: utf-8~n~p.~n", [app]) - File.write!("my_sample_mode/ebin/my_sample_mode.app", format) + ) apps = [my_sample_mode: :temporary] release = release(applications: apps) @@ -802,18 +799,15 @@ defmodule Mix.ReleaseTest do describe "included applications" do test "are included in the release", context do in_tmp(context.test, fn -> - app = + write_app!( + "my_sample1/ebin/my_sample1.app", {:application, :my_sample1, applications: [:kernel, :stdlib, :elixir], description: ~c"my_sample1", modules: [], vsn: ~c"1.0.0", included_applications: [:runtime_tools]} - - File.mkdir_p!("my_sample1/ebin") - Code.prepend_path("my_sample1/ebin") - format = :io_lib.format("%% coding: utf-8~n~p.~n", [app]) - File.write!("my_sample1/ebin/my_sample1.app", format) + ) release = release(applications: [my_sample1: :permanent]) assert release.boot_scripts.start[:runtime_tools] == :load @@ -825,18 +819,15 @@ defmodule Mix.ReleaseTest do test "raise on conflict", context do in_tmp(context.test, fn -> - app = + write_app!( + "my_sample2/ebin/my_sample2.app", {:application, :my_sample2, applications: [:kernel, :stdlib, :elixir, :runtime_tools], description: ~c"my_sample", modules: [], vsn: ~c"1.0.0", included_applications: [:runtime_tools]} - - File.mkdir_p!("my_sample2/ebin") - Code.prepend_path("my_sample2/ebin") - format = :io_lib.format("%% coding: utf-8~n~p.~n", [app]) - File.write!("my_sample2/ebin/my_sample2.app", format) + ) assert_raise Mix.Error, ":runtime_tools is listed both as a regular application and as an included application", @@ -848,25 +839,72 @@ defmodule Mix.ReleaseTest do describe "optional applications" do test "are ignored if not available", context do in_tmp(context.test, fn -> - app = - {:application, :my_sample1, + write_app!( + "my_sample3/ebin/my_sample3.app", + {:application, :my_sample3, applications: [:kernel, :stdlib, :elixir, :unknown], optional_applications: [:unknown], - description: ~c"my_sample1", + description: ~c"my_sample3", modules: [], vsn: ~c"1.0.0"} + ) - File.mkdir_p!("my_sample1/ebin") - Code.prepend_path("my_sample1/ebin") - format = :io_lib.format("%% coding: utf-8~n~p.~n", [app]) - File.write!("my_sample1/ebin/my_sample1.app", format) + release = release(applications: [my_sample3: :permanent]) + assert release.boot_scripts.start[:unknown] == nil + end) + end - release = release(applications: [my_sample1: :permanent]) + test "are ignored even if mode changes", context do + in_tmp(context.test, fn -> + write_app!( + "has_optional/ebin/has_optional.app", + {:application, :has_optional, + applications: [:kernel, :stdlib, :elixir, :unknown], + optional_applications: [:unknown], + description: ~c"has_optional", + modules: [], + vsn: ~c"1.0.0"} + ) + + write_app!( + "points_as_permanent/ebin/points_as_permanent.app", + {:application, :points_as_permanent, + applications: [:kernel, :stdlib, :elixir, :has_optional], + optional_applications: [:unknown], + description: ~c"points_as_permanent", + modules: [], + vsn: ~c"1.0.0"} + ) + + write_app!( + "points_as_temporary/ebin/points_as_temporary.app", + {:application, :points_as_temporary, + applications: [:kernel, :stdlib, :elixir, :has_optional], + optional_applications: [:unknown], + description: ~c"points_as_temporary", + modules: [], + vsn: ~c"1.0.0"} + ) + + release = + release( + applications: [points_as_permanent: :permanent, points_as_temporary: :temporary] + ) + + assert release.boot_scripts.start[:has_optional] == :permanent assert release.boot_scripts.start[:unknown] == nil end) end end + defp write_app!(path, app) do + dir = Path.dirname(path) + File.mkdir_p!(dir) + Code.prepend_path(dir) + format = :io_lib.format("%% coding: utf-8~n~p.~n", [app]) + File.write!(path, format) + end + defp size!(path) do File.stat!(path).size end diff --git a/lib/mix/test/mix/tasks/archive_test.exs b/lib/mix/test/mix/tasks/archive_test.exs index 35d8a811788..5d648be104a 100644 --- a/lib/mix/test/mix/tasks/archive_test.exs +++ b/lib/mix/test/mix/tasks/archive_test.exs @@ -32,6 +32,7 @@ defmodule Mix.Tasks.ArchiveTest do message = "Generated archive \"archive-0.1.0.ez\" with MIX_ENV=dev" assert_received {:mix_shell, :info, [^message]} assert File.regular?(~c"archive-0.1.0.ez") + assert to_charlist(Mix.Project.consolidation_path()) not in :code.get_path() assert_archive_content_default() refute has_in_zip_file?(~c"archive-0.1.0.ez", ~c"archive-0.1.0/priv/.dot_file") diff --git a/lib/mix/test/mix/tasks/compile.elixir_test.exs b/lib/mix/test/mix/tasks/compile.elixir_test.exs index 06809ad4be3..94afaeb86be 100644 --- a/lib/mix/test/mix/tasks/compile.elixir_test.exs +++ b/lib/mix/test/mix/tasks/compile.elixir_test.exs @@ -19,6 +19,14 @@ defmodule Mix.Tasks.Compile.ElixirTest do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) + + File.write!("lib/a.ex", """ + defmodule A, do: :ok + + # Also make sure that we access the ebin directory during compilation + true = to_charlist(Mix.Project.compile_path()) in :code.get_path() + """) + Mix.Tasks.Compile.Elixir.run(["--verbose"]) assert File.regular?("_build/shared/lib/sample/ebin/Elixir.A.beam") @@ -32,6 +40,14 @@ defmodule Mix.Tasks.Compile.ElixirTest do test "compiles a project with per environment build" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) + + File.write!("lib/a.ex", """ + defmodule A, do: :ok + + # Also make sure that we access the ebin directory during compilation + true = to_charlist(Mix.Project.compile_path()) in :code.get_path() + """) + Mix.Tasks.Compile.Elixir.run(["--verbose"]) assert File.regular?("_build/dev/lib/sample/ebin/Elixir.A.beam") @@ -772,7 +788,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do end) end - test "compiles mtime changed files if content changed but not length" do + test "recompiles mtime changed files if content changed but not length" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == {:ok, []} @@ -872,7 +888,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do end) end - test "compiles size changed files" do + test "recompiles size changed files" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) past = @old_time @@ -894,7 +910,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do end) end - test "compiles dependent changed modules" do + test "recompiles dependent changed modules" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) File.write!("lib/a.ex", "defmodule A, do: B.module_info()") @@ -914,7 +930,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do end) end - test "compiles dependent changed modules without beam files" do + test "recompiles dependent changed modules without beam files" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) @@ -942,7 +958,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do Code.put_compiler_option(:ignore_module_conflict, false) end - test "compiles dependent changed modules even on removal" do + test "recompiles dependent changed modules even on removal" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) File.write!("lib/a.ex", "defmodule A, do: B.module_info()") @@ -963,7 +979,7 @@ defmodule Mix.Tasks.Compile.ElixirTest do end) end - test "compiles dependent changed external resources" do + test "recompiles dependent changed external resources" do in_fixture("no_mixfile", fn -> Mix.Project.push(MixTest.Case.Sample) tmp = tmp_path("c.eex") @@ -1263,6 +1279,11 @@ defmodule Mix.Tasks.Compile.ElixirTest do assert_received {:mix_shell, :info, ["Compiled lib/b.ex"]} assert_received {:mix_shell, :info, ["Compiled lib/c.ex"]} + # Mix recompile should work even if the compile path + # was removed and the module purged + Code.delete_path(Mix.Project.compile_path()) + purge([A]) + assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == {:ok, []} assert_received {:mix_shell, :info, ["Compiling 1 file (.ex)"]} assert_received {:mix_shell, :info, ["Compiled lib/a.ex"]} diff --git a/lib/mix/test/mix/tasks/compile.yecc_test.exs b/lib/mix/test/mix/tasks/compile.yecc_test.exs index 859ff5417a6..9b7d3474db9 100644 --- a/lib/mix/test/mix/tasks/compile.yecc_test.exs +++ b/lib/mix/test/mix/tasks/compile.yecc_test.exs @@ -60,6 +60,23 @@ defmodule Mix.Tasks.Compile.YeccTest do severity: :warning } = diagnostic end) + + # Removing parse tools re-add it later even if only to show warnings + :code.del_path(:parsetools) + purge([:yecc]) + refute Code.ensure_loaded?(:yecc) + + capture_io(fn -> + assert {:noop, [diagnostic]} = Mix.Tasks.Compile.Yecc.run([]) + + assert %Mix.Task.Compiler.Diagnostic{ + compiler_name: "yecc", + file: ^file, + message: "conflicts: 1 shift/reduce, 0 reduce/reduce", + position: 0, + severity: :warning + } = diagnostic + end) end) end diff --git a/lib/mix/test/mix/tasks/compile_test.exs b/lib/mix/test/mix/tasks/compile_test.exs index 5c3a1b5ea1d..e0a7247525f 100644 --- a/lib/mix/test/mix/tasks/compile_test.exs +++ b/lib/mix/test/mix/tasks/compile_test.exs @@ -5,12 +5,6 @@ defmodule Mix.Tasks.CompileTest do defmacro position(line, column), do: {line, column} - defmodule CustomCompilers do - def project do - [compilers: [:elixir, :app, :custom]] - end - end - defmodule DepsApp do def project do [app: :deps_app, version: "0.1.0", deps: [{:ok, "0.1.0", path: "deps/ok"}]] @@ -23,7 +17,8 @@ defmodule Mix.Tasks.CompileTest do end end - setup do + setup tags do + Mix.ProjectStack.post_config(Map.get(tags, :project, [])) Mix.Project.push(MixTest.Case.Sample) :ok end @@ -37,16 +32,14 @@ defmodule Mix.Tasks.CompileTest do assert_received {:mix_shell, :info, ["mix compile.elixir # " <> _]} end + @tag project: [compilers: [:elixir, :app, :custom]] test "compiles --list with custom mixfile" do - Mix.Project.pop() - Mix.Project.push(CustomCompilers) Mix.Task.run("compile", ["--list"]) assert_received {:mix_shell, :info, ["\nEnabled compilers: elixir, app, custom, protocols"]} end + @tag project: [compilers: [:elixir, :app, :custom]] test "compiles does not require all compilers available on manifest" do - Mix.Project.pop() - Mix.Project.push(CustomCompilers) assert Mix.Tasks.Compile.manifests() |> Enum.map(&Path.basename/1) == ["compile.elixir"] end @@ -101,6 +94,34 @@ defmodule Mix.Tasks.CompileTest do end) end + @tag project: [compilers: Mix.compilers() ++ [:my_custom_compiler]] + test "compiles a project with custom in-project compiler" do + in_fixture("no_mixfile", fn -> + File.mkdir_p!("lib") + + File.write!("lib/a.ex", """ + defmodule Mix.Tasks.Compile.MyCustomCompiler do + use Mix.Task.Compiler + + @impl true + def run(_args) do + Mix.shell().info("Compiling...") + :ok + end + end + """) + + assert Mix.Task.run("compile") == {:ok, []} + assert_receive {:mix_shell, :info, ["Compiling..."]} + Code.delete_path(Mix.Project.compile_path()) + purge([Mix.Tasks.Compile.MyCustomCompiler]) + false = Code.ensure_loaded?(Mix.Tasks.Compile.MyCustomCompiler) + + Mix.Task.clear() + assert Mix.Task.rerun("compile") + end) + end + test "recompiles app cache if manifest changes" do in_fixture("no_mixfile", fn -> Mix.Tasks.Compile.run(["--force"]) @@ -329,12 +350,22 @@ defmodule Mix.Tasks.CompileTest do Application.delete_env(:sample, :hello, persistent: true) end - test "code path prunning" do + test "code path pruning" do Mix.ensure_application!(:parsetools) + otp_docs? = match?({:docs_v1, _, _, _, _, _, _}, Code.fetch_docs(:zlib)) in_fixture("no_mixfile", fn -> assert Mix.Task.run("compile", []) == {:ok, []} assert :code.where_is_file(~c"parsetools.app") == :non_existing + + # Make sure erts is also kept but not loaded + assert Application.spec(:erts, :vsn) == nil + + if otp_docs? do + assert {:docs_v1, _, _, _, _, _, _} = Code.fetch_docs(:zlib) + else + IO.warn("Erlang/OTP was not compiled with docs, skipping assertion") + end end) end diff --git a/lib/mix/test/mix/tasks/format_test.exs b/lib/mix/test/mix/tasks/format_test.exs index 254ac47c90b..349e4026da6 100644 --- a/lib/mix/test/mix/tasks/format_test.exs +++ b/lib/mix/test/mix/tasks/format_test.exs @@ -308,6 +308,24 @@ defmodule Mix.Tasks.FormatTest do '''abc end """ + + {formatter_function, _options} = Mix.Tasks.Format.formatter_for_file("a.ex") + + assert formatter_function.(""" + if true do + ~W''' + foo bar baz + '''abc + end + """) == """ + if true do + ~W''' + foo + bar + baz + '''abc + end + """ end) end @@ -555,11 +573,18 @@ defmodule Mix.Tasks.FormatTest do test "reads exported configuration from subdirectories", context do in_tmp(context.test, fn -> File.write!(".formatter.exs", """ - [subdirectories: ["lib"]] + [subdirectories: ["li", "lib"]] """) + # We also create a directory called li to ensure files + # from lib won't accidentally match on li. + File.mkdir_p!("li") File.mkdir_p!("lib") + File.write!("li/.formatter.exs", """ + [inputs: "**/*", locals_without_parens: [other_fun: 2]] + """) + File.write!("lib/.formatter.exs", """ [inputs: "a.ex", locals_without_parens: [my_fun: 2]] """) diff --git a/lib/mix/test/mix/tasks/xref_test.exs b/lib/mix/test/mix/tasks/xref_test.exs index c025ff142ab..fa6b0de9ea9 100644 --- a/lib/mix/test/mix/tasks/xref_test.exs +++ b/lib/mix/test/mix/tasks/xref_test.exs @@ -915,6 +915,43 @@ defmodule Mix.Tasks.XrefTest do end) end + test "generates reports from the umbrella root" do + Mix.Project.pop() + + in_fixture("umbrella_dep/deps/umbrella", fn -> + Mix.Project.in_project(:umbrella, ".", fn _ -> + File.write!("apps/bar/lib/bar.ex", """ + defmodule Bar do + def bar do + Foo.foo() + end + end + """) + + Mix.Task.run("compile") + Mix.shell().flush() + + Mix.Tasks.Xref.run(["graph", "--format", "stats", "--include-siblings"]) + + assert receive_until_no_messages([]) == """ + Tracked files: 2 (nodes) + Compile dependencies: 0 (edges) + Exports dependencies: 0 (edges) + Runtime dependencies: 1 (edges) + Cycles: 0 + + Top 2 files with most outgoing dependencies: + * lib/bar.ex (1) + * lib/foo.ex (0) + + Top 2 files with most incoming dependencies: + * lib/foo.ex (1) + * lib/bar.ex (0) + """ + end) + end) + end + test "generates reports considering siblings inside umbrellas" do Mix.Project.pop() diff --git a/lib/mix/test/mix/umbrella_test.exs b/lib/mix/test/mix/umbrella_test.exs index d5d0c5922ce..2561bd297bc 100644 --- a/lib/mix/test/mix/umbrella_test.exs +++ b/lib/mix/test/mix/umbrella_test.exs @@ -64,6 +64,10 @@ defmodule Mix.UmbrellaTest do Mix.Task.run("deps.loadpaths") Mix.Task.run("compile", ["--verbose"]) + # Extra applications are picked even for umbrellas + assert :code.where_is_file(~c"runtime_tools.app") != :non_existing + assert :code.where_is_file(~c"observer.app") == :non_existing + assert_received {:mix_shell, :info, ["==> bar"]} assert_received {:mix_shell, :info, ["Generated bar app"]} assert File.regular?("_build/dev/lib/bar/ebin/Elixir.Bar.beam") @@ -252,8 +256,6 @@ defmodule Mix.UmbrellaTest do end) end - ## Umbrellas as a dependency - test "list deps for umbrella as dependency" do in_fixture("umbrella_dep", fn -> Mix.Project.in_project(:umbrella_dep, ".", fn _ -> @@ -292,7 +294,7 @@ defmodule Mix.UmbrellaTest do in_fixture("umbrella_dep", fn -> Mix.Project.push(CycleDeps) - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:foo, :bar, :umbrella] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:foo, :bar, :umbrella] end) end @@ -330,7 +332,7 @@ defmodule Mix.UmbrellaTest do end """) - assert Enum.map(Mix.Dep.load_on_environment([]), & &1.app) == [:a, :b, :bar, :foo] + assert Enum.map(Mix.Dep.Converger.converge([]), & &1.app) == [:a, :b, :bar, :foo] end) end) end @@ -373,7 +375,7 @@ defmodule Mix.UmbrellaTest do end) end - test "recompiles after compile time path dependency changes" do + test "recompiles when compile-time path dependencies change" do in_fixture("umbrella_dep/deps/umbrella/apps", fn -> Mix.Project.in_project(:bar, "bar", fn _ -> Mix.Task.run("compile", []) @@ -385,30 +387,13 @@ defmodule Mix.UmbrellaTest do assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} assert_receive {:mix_shell, :info, ["Compiled lib/bar.ex"]} - # Emulate local recompilation + # Compile-time dependencies are recompiled File.write!("../foo/lib/foo.ex", File.read!("../foo/lib/foo.ex") <> "\n") - mtime = File.stat!("_build/dev/lib/bar/.mix/compile.elixir").mtime - ensure_touched("../foo/lib/foo.ex", mtime) - - Mix.Task.clear() - assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} - assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} - - # But exports dependencies are not recompiled - File.write!("lib/bar.ex", "defmodule Bar, do: (require Foo)") + ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir") Mix.Task.clear() assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} - - # Touch to emulate local recompilation - File.write!("../foo/lib/foo.ex", File.read!("../foo/lib/foo.ex") <> "\n") - mtime = File.stat!("_build/dev/lib/bar/.mix/compile.elixir").mtime - ensure_touched("../foo/lib/foo.ex", mtime) - - Mix.Task.clear() - assert Mix.Tasks.Compile.Elixir.run(["--verbose"]) == {:noop, []} - refute_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} end) # Now let's add a new file to foo @@ -446,18 +431,40 @@ defmodule Mix.UmbrellaTest do File.write!("../foo/lib/foo.ex", "defmodule Foo, do: defstruct [:bar]") # Add struct dependency - File.write!("lib/bar.ex", "defmodule Bar, do: %Foo{bar: true}") + File.write!("lib/bar.ex", """ + defmodule Bar do + def is_foo_bar(%Foo{bar: true}), do: true + end + """) + Mix.Task.run("compile", ["--verbose"]) - assert_receive {:mix_shell, :info, ["Compiled lib/bar.ex"]} + assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} - # Recompiles for struct dependencies + # Does not recompiles if export dependency does not change File.write!("../foo/lib/foo.ex", File.read!("../foo/lib/foo.ex") <> "\n") - mtime = File.stat!("_build/dev/lib/bar/.mix/compile.elixir").mtime - ensure_touched("../foo/lib/foo.ex", mtime) + ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir") Mix.Task.clear() assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} - assert_receive {:mix_shell, :info, ["Compiled lib/bar.ex"]} + refute_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} + + # Recompiles if export dependency changes + File.write!("../foo/lib/foo.ex", "defmodule Foo, do: defstruct [:bar, :baz]") + ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir") + + Mix.Task.clear() + assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} + assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} + + # Recompiles if export dependency is removed + File.write!("../foo/lib/foo.ex", "") + ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir") + Mix.Task.clear() + + ExUnit.CaptureIO.capture_io(:stderr, fn -> + Process.flag(:trap_exit, true) + catch_exit(Mix.Task.run("compile", ["--verbose"])) + end) end) end) end @@ -480,16 +487,15 @@ defmodule Mix.UmbrellaTest do # Add compile time to Foo.Bar File.write!("lib/bar.ex", "defmodule Bar, do: Foo.Bar.hello()") Mix.Task.run("compile", ["--verbose"]) - assert_receive {:mix_shell, :info, ["Compiled lib/bar.ex"]} + assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} # Recompiles for due to compile dependency via runtime dependencies File.write!("../foo/lib/foo.baz.ex", File.read!("../foo/lib/foo.baz.ex") <> "\n") - mtime = File.stat!("_build/dev/lib/bar/.mix/compile.elixir").mtime - ensure_touched("../foo/lib/foo.ex", mtime) + ensure_touched("../foo/lib/foo.ex", "_build/dev/lib/bar/.mix/compile.elixir") Mix.Task.clear() assert Mix.Task.run("compile", ["--verbose"]) == {:ok, []} - assert_receive {:mix_shell, :info, ["Compiled lib/bar.ex"]} + assert_received {:mix_shell, :info, ["Compiled lib/bar.ex"]} end) end) end diff --git a/lib/mix/test/test_helper.exs b/lib/mix/test/test_helper.exs index 0f7b02a5413..61733a02c5f 100644 --- a/lib/mix/test/test_helper.exs +++ b/lib/mix/test/test_helper.exs @@ -1,3 +1,16 @@ +home = Path.expand("../tmp/.home", __DIR__) +File.mkdir_p!(home) +System.put_env("HOME", home) + +mix = Path.expand("../tmp/.mix", __DIR__) +File.mkdir_p!(mix) +System.put_env("MIX_HOME", mix) + +System.delete_env("XDG_DATA_HOME") +System.delete_env("XDG_CONFIG_HOME") + +## Setup Mix + Mix.start() Mix.shell(Mix.Shell.Process) Application.put_env(:mix, :colors, enabled: false) @@ -5,6 +18,8 @@ Application.put_env(:mix, :colors, enabled: false) Logger.remove_backend(:console) Application.put_env(:logger, :backends, []) +## Setup ExUnit + os_exclude = if match?({:win32, _}, :os.type()), do: [unix: true], else: [windows: true] epmd_exclude = if match?({:win32, _}, :os.type()), do: [epmd: true], else: [] git_exclude = if Mix.SCM.Git.git_version() <= {1, 7, 4}, do: [git_sparse: true], else: [] @@ -212,18 +227,7 @@ defmodule MixTest.Case do end end -## Set up globals - -home = MixTest.Case.tmp_path(".home") -File.mkdir_p!(home) -System.put_env("HOME", home) - -mix = MixTest.Case.tmp_path(".mix") -File.mkdir_p!(mix) -System.put_env("MIX_HOME", mix) - -System.delete_env("XDG_DATA_HOME") -System.delete_env("XDG_CONFIG_HOME") +## Set up Rebar fixtures rebar3_source = System.get_env("REBAR3") || Path.expand("fixtures/rebar3", __DIR__) [major, minor | _] = String.split(System.version(), ".") @@ -231,8 +235,6 @@ rebar3_target = Path.join([mix, "elixir", "#{major}-#{minor}", "rebar3"]) File.mkdir_p!(Path.dirname(rebar3_target)) File.cp!(rebar3_source, rebar3_target) -## Copy fixtures to tmp - fixtures = ~w(rebar_dep rebar_override) Enum.each(fixtures, fn fixture -> @@ -242,12 +244,13 @@ Enum.each(fixtures, fn fixture -> File.cp_r!(source, dest) end) -## Generate Git repo fixtures +## Set up Git fixtures + System.cmd("git", ~w[config --global user.email mix@example.com]) System.cmd("git", ~w[config --global user.name mix-repo]) System.cmd("git", ~w[config --global init.defaultBranch not-main]) -# Git repo +### Git repo target = Path.expand("fixtures/git_repo", __DIR__) unless File.dir?(target) do @@ -329,7 +332,7 @@ unless File.dir?(target) do end) end -# Deps on Git repo +### Deps on Git repo target = Path.expand("fixtures/deps_on_git_repo", __DIR__) unless File.dir?(target) do @@ -423,7 +426,7 @@ Enum.each([:invalidapp, :invalidvsn, :noappfile, :nosemver, :ok], fn dep -> File.mkdir_p!(Path.expand("fixtures/deps_status/deps/#{dep}/.git", __DIR__)) end) -# Archive ebin +### Archive ebin target = Path.expand("fixtures/archive", __DIR__) unless File.dir?(Path.join(target, "ebin")) do