Merge pull request #4062 from emqx/dev/v4.3.0

Auto-pull-request-on-2021-01-24
This commit is contained in:
Zaiming Shi 2021-01-24 22:29:09 +01:00 committed by GitHub
commit 20c37609af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 495 additions and 210 deletions

View File

@ -25,6 +25,7 @@ emqx_test(){
unzip -q ${PACKAGE_PATH}/$packagename
sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" ${PACKAGE_PATH}/emqx/etc/emqx.conf
sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" ${PACKAGE_PATH}/emqx/etc/emqx.conf
sed -i '/emqx_telemetry/d' ${PACKAGE_PATH}/emqx/data/loaded_plugins
if [ ! -z $(echo ${EMQX_DEPS_DEFAULT_VSN#v} | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?-(alpha|beta|rc)\.[0-9]") ]; then
if [ ! -d ${PACKAGE_PATH}/emqx/lib/emqx-${EMQX_DEPS_DEFAULT_VSN#v} ] || [ ! -d ${PACKAGE_PATH}/emqx/releases/${EMQX_DEPS_DEFAULT_VSN#v} ] ;then
@ -111,6 +112,7 @@ running_test(){
sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" /etc/emqx/emqx.conf
sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" /etc/emqx/emqx.conf
sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
emqx start || tail /var/log/emqx/erlang.log.1
IDLE_TIME=0

View File

@ -16,6 +16,7 @@ services:
- -c
- |
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
/usr/bin/start.sh
healthcheck:
test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
@ -42,6 +43,7 @@ services:
- -c
- |
sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
/usr/bin/start.sh
healthcheck:
test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]

View File

@ -26,6 +26,7 @@
!cd emqx
!sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
!sed -i '/emqx_telemetry/d' data/loaded_plugins
!./bin/emqx start
?EMQ X Broker $old_vsn is started successfully!
@ -42,6 +43,7 @@
?SH-PROMPT
!cd emqx2
!sed -i '/emqx_telemetry/d' data/loaded_plugins
!./bin/emqx start
?EMQ X Broker $old_vsn is started successfully!

View File

@ -58,7 +58,7 @@ ignore=title-trailing-punctuation, T1, T2, T3, T4, T5, T6, T8, B1, B2, B3, B4, B
# python-style regex that the commit-msg title must match
# Note that the regex can contradict with other rules if not used correctly
# (e.g. title-must-not-contain-word).
regex=^(feat|feature|fix|docs|style|refactor|test|chore|perf|improve)\(.+\): .+
regex=^(feat|feature|fix|docs|style|refactor|test|chore|build|perf|improve)\(.+\): .+
# [body-max-line-length]
# line-length=72

View File

@ -91,6 +91,7 @@ jobs:
run: |
pkg_name=$(basename _packages/emqx/emqx-macos-*.zip)
unzip _packages/emqx/$pkg_name
gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
./emqx/bin/emqx start || cat emqx/log/erlang.log.1
./emqx/bin/emqx_ctl status
./emqx/bin/emqx stop

View File

@ -77,6 +77,7 @@ jobs:
sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
sed -i -r 's/ pullPolicy: .*$/ pullPolicy: Never/g' deploy/charts/emqx/values.yaml
sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx --debug --dry-run
helm install emqx --set emqxAclConfig="" --set emqxConfig.EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s --set emqxConfig.EMQX_MQTT__MAX_TOPIC_ALIAS=10 deploy/charts/emqx
@ -89,6 +90,11 @@ jobs:
echo "waiting emqx started";
sleep 10;
done
- name: get pods log
if: failure()
env:
KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
run: kubectl describe pods emqx-0
- uses: actions/checkout@v2
with:
repository: emqx/paho.mqtt.testing

View File

@ -47,6 +47,14 @@ auth.pgsql.ssl.enable = off
##
## auth.pgsql.ssl_opts.tls_versions = tlsv1.2
## TLS version
## You can configure multi-version use "," split,
## default value is :tlsv1.2
## Example:
## tlsv1.1,tlsv1.2,tlsv1.3
##
## auth.pgsql.ssl.tls_versions = tlsv1.2
## SSL keyfile.
##
## Value: File

View File

@ -67,6 +67,12 @@
{datatype, string}
]}.
%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0
{mapping, "auth.pgsql.ssl_opts.tls_versions", "emqx_auth_pgsql.server", [
{default, "tlsv1.2"},
{datatype, string}
]}.
{translation, "emqx_auth_pgsql.server", fun(Conf) ->
{PgHost, PgPort} =
case cuttlefish:conf_get("auth.pgsql.server", Conf) of
@ -88,7 +94,7 @@
{certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
{cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined),
{versions, [list_to_existing_atom(Value)
||Value <- string:tokens(cuttlefish:conf_get("auth.pgsql.ssl.tls_versions", Conf), " ,")]}}])
||Value <- string:tokens(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf), " ,")]}}])
end,
%% FIXME: compatible with 4.0-4.2 version format, plan to delete in 5.0

View File

@ -241,7 +241,7 @@ handle_received_publish(Topic, MaxAge, Format, Payload) ->
handle_received_create(TopicPrefix, MaxAge, Payload) ->
case core_link:decode(Payload) of
[{rootless, [Topic], [{ct, CT}]}] when is_binary(Topic), Topic =/= <<>> ->
TrueTopic = http_uri:decode(Topic),
TrueTopic = percent_decode(Topic),
?LOG(debug, "decoded link-format payload, the Topic=~p, CT=~p~n", [TrueTopic, CT]),
LocPath = concatenate_location_path([<<"ps">>, TopicPrefix, TrueTopic]),
FullTopic = binary:part(LocPath, 4, byte_size(LocPath)-4),
@ -259,6 +259,14 @@ handle_received_create(TopicPrefix, MaxAge, Payload) ->
{error, bad_request}
end.
%% http_uri:decode/1 is deprecated in OTP-23
%% its equivalent uri_string:percent_decode however is not available before OTP 23
-if(?OTP_RELEASE >= 23).
percent_decode(Topic) -> uri_string:percent_decode(Topic).
-else.
percent_decode(Topic) -> http_uri:decode(Topic).
-endif.
%% When topic is timeout, server should return nocontent here,
%% but gen_coap only receive return value of #coap_content from coap_get, so temporarily we can't give the Code 2.07 {ok, nocontent} out.TBC!!!
return_resource(Topic, Payload, MaxAge, TimeStamp, Content) ->

View File

@ -83,6 +83,7 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
- "register": LwM2M Register
- "update": LwM2M Update
- "data" contains the query options and the object-list of the register message
- The *update* message is only published if the object-list changed.
### Downlink Command and Uplink Response (LwM2M Device Management & Service Enablement Interface)
@ -113,6 +114,7 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
"path": {?ResourcePath}
}
```
- {?ResourcePath}: String, LwM2M full resource path. e.g. "3/0", "/3/0/0", "/3/0/6/0"
- **If {?MsgType} = "write" (single write)**:
```json
{
@ -121,6 +123,8 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
"value": {?Value}
}
```
- {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk"
- {?Value}: Value of the resource, depends on "type".
- **If {?MsgType} = "write" (batch write)**:
```json
{
@ -183,21 +187,6 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
}
```
- {?ObjectInstanceID}: Integer, LwM2M Object Instance ID
- {?ResourcePath}: String, LwM2M full resource path. e.g. "3/0", "/3/0/0", "/3/0/6/0"
- {?Content}:
```json
[
{
"name": {?ResourceName},
"path": {?ResourcePath},
"type": {?ValueType},
"value": {?Value}
}
]
```
- {?ResourceName}: String, LwM2M resource description, e.g. "Available Power Sources"
- {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk"
- {?Value}: Value of resource, depends on the {?ValueType}.
- **The response of LwM2M will be converted to following MQTT message:**
- **Method:** PUBLISH
@ -262,17 +251,11 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
```json
[
{
"name": {?ResourceName},
"path": {?ResourcePath},
"type": {?ValueType},
"value": {?Value}
}
]
```
- {?ResourceName}: String, LwM2M resource description, e.g. "Available Power Sources"
- {?ResourcePath}: String, LwM2M resource full path. e.g. "3/0", "/3/0/0", "/3/0/6/0"
- {?ValueType}: String, can be: "Time", "String", "Integer", "Float", "Boolean", "Opaque", "Objlnk"
- {?Value}: Value of the resource, depends on "type".
- **If {?MsgType} = "ack", "data" does not exists**
@ -308,11 +291,10 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
"data": {
"code": {?StatusCode},
"codeMsg": {?CodeMsg},
"reqPath": {?RequestPath},
"content": [
{
"name": {?ResourceName},
"path": {?ResourcePath},
"type": {?ValueType},
"value": {?Value}
}
]
@ -338,11 +320,12 @@ The MQTT message will be translated to an LwM2M DISCOVER command and sent to the
"data": {
"code": {?StatusCode},
"codeMsg": {?CodeMsg},
"reqPath": {?RequestPath},
"content": [
"name": {?ResourceName},
"path": {?ResourcePath},
"type": {?ValueType},
"value": {?Value}
{
"path": {?ResourcePath},
"value": {?Value}
}
]
}
}

View File

@ -48,6 +48,229 @@ json data
```
## Before EMQ X v4.0.0
The prometheus data simple is:
```bash
# TYPE erlang_vm_ets_limit gauge
erlang_vm_ets_limit 2097152
# TYPE erlang_vm_logical_processors gauge
erlang_vm_logical_processors 2
# TYPE erlang_vm_logical_processors_available gauge
erlang_vm_logical_processors_available 2
# TYPE erlang_vm_logical_processors_online gauge
erlang_vm_logical_processors_online 2
# TYPE erlang_vm_port_count gauge
erlang_vm_port_count 19
# TYPE erlang_vm_port_limit gauge
erlang_vm_port_limit 1048576
# TYPE erlang_vm_process_count gauge
erlang_vm_process_count 460
# TYPE erlang_vm_process_limit gauge
erlang_vm_process_limit 2097152
# TYPE erlang_vm_schedulers gauge
erlang_vm_schedulers 2
# TYPE erlang_vm_schedulers_online gauge
erlang_vm_schedulers_online 2
# TYPE erlang_vm_smp_support untyped
erlang_vm_smp_support 1
# TYPE erlang_vm_threads untyped
erlang_vm_threads 1
# TYPE erlang_vm_thread_pool_size gauge
erlang_vm_thread_pool_size 32
# TYPE erlang_vm_time_correction untyped
erlang_vm_time_correction 1
# TYPE erlang_vm_statistics_context_switches counter
erlang_vm_statistics_context_switches 39850
# TYPE erlang_vm_statistics_garbage_collection_number_of_gcs counter
erlang_vm_statistics_garbage_collection_number_of_gcs 17116
# TYPE erlang_vm_statistics_garbage_collection_words_reclaimed counter
erlang_vm_statistics_garbage_collection_words_reclaimed 55711819
# TYPE erlang_vm_statistics_garbage_collection_bytes_reclaimed counter
erlang_vm_statistics_garbage_collection_bytes_reclaimed 445694552
# TYPE erlang_vm_statistics_bytes_received_total counter
erlang_vm_statistics_bytes_received_total 400746
# TYPE erlang_vm_statistics_bytes_output_total counter
erlang_vm_statistics_bytes_output_total 337197
# TYPE erlang_vm_statistics_reductions_total counter
erlang_vm_statistics_reductions_total 21157980
# TYPE erlang_vm_statistics_run_queues_length_total gauge
erlang_vm_statistics_run_queues_length_total 0
# TYPE erlang_vm_statistics_runtime_milliseconds counter
erlang_vm_statistics_runtime_milliseconds 6559
# TYPE erlang_vm_statistics_wallclock_time_milliseconds counter
erlang_vm_statistics_wallclock_time_milliseconds 261243
# TYPE erlang_vm_memory_atom_bytes_total gauge
erlang_vm_memory_atom_bytes_total{usage="used"} 1814822
erlang_vm_memory_atom_bytes_total{usage="free"} 22459
# TYPE erlang_vm_memory_bytes_total gauge
erlang_vm_memory_bytes_total{kind="system"} 109820104
erlang_vm_memory_bytes_total{kind="processes"} 44983656
# TYPE erlang_vm_dets_tables gauge
erlang_vm_dets_tables 1
# TYPE erlang_vm_ets_tables gauge
erlang_vm_ets_tables 139
# TYPE erlang_vm_memory_processes_bytes_total gauge
erlang_vm_memory_processes_bytes_total{usage="used"} 44983656
erlang_vm_memory_processes_bytes_total{usage="free"} 0
# TYPE erlang_vm_memory_system_bytes_total gauge
erlang_vm_memory_system_bytes_total{usage="atom"} 1837281
erlang_vm_memory_system_bytes_total{usage="binary"} 595872
erlang_vm_memory_system_bytes_total{usage="code"} 40790577
erlang_vm_memory_system_bytes_total{usage="ets"} 37426896
erlang_vm_memory_system_bytes_total{usage="other"} 29169478
# TYPE erlang_mnesia_held_locks gauge
erlang_mnesia_held_locks 0
# TYPE erlang_mnesia_lock_queue gauge
erlang_mnesia_lock_queue 0
# TYPE erlang_mnesia_transaction_participants gauge
erlang_mnesia_transaction_participants 0
# TYPE erlang_mnesia_transaction_coordinators gauge
erlang_mnesia_transaction_coordinators 0
# TYPE erlang_mnesia_failed_transactions counter
erlang_mnesia_failed_transactions 2
# TYPE erlang_mnesia_committed_transactions counter
erlang_mnesia_committed_transactions 239
# TYPE erlang_mnesia_logged_transactions counter
erlang_mnesia_logged_transactions 60
# TYPE erlang_mnesia_restarted_transactions counter
erlang_mnesia_restarted_transactions 0
# TYPE emqx_packets_auth_received counter
emqx_packets_auth_received 0
# TYPE emqx_packets_auth_sent counter
emqx_packets_auth_sent 0
# TYPE emqx_packets_received counter
emqx_packets_received 0
# TYPE emqx_packets_sent counter
emqx_packets_sent 0
# TYPE emqx_packets_connect counter
emqx_packets_connect 0
# TYPE emqx_packets_connack_sent counter
emqx_packets_connack_sent 0
# TYPE emqx_packets_connack_error counter
emqx_packets_connack_error 0
# TYPE emqx_packets_connack_auth_error counter
emqx_packets_connack_auth_error 0
# TYPE emqx_packets_disconnect_received counter
emqx_packets_disconnect_received 0
# TYPE emqx_packets_disconnect_sent counter
emqx_packets_disconnect_sent 0
# TYPE emqx_packets_subscribe counter
emqx_packets_subscribe 0
# TYPE emqx_packets_subscribe_error counter
emqx_packets_subscribe_error 0
# TYPE emqx_packets_subscribe_auth_error counter
emqx_packets_subscribe_auth_error 0
# TYPE emqx_packets_suback counter
emqx_packets_suback 0
# TYPE emqx_packets_unsubscribe counter
emqx_packets_unsubscribe 0
# TYPE emqx_packets_unsubscribe_error counter
emqx_packets_unsubscribe_error 0
# TYPE emqx_packets_unsuback counter
emqx_packets_unsuback 0
# TYPE emqx_packets_publish_received counter
emqx_packets_publish_received 0
# TYPE emqx_packets_publish_sent counter
emqx_packets_publish_sent 0
# TYPE emqx_packets_publish_auth_error counter
emqx_packets_publish_auth_error 0
# TYPE emqx_packets_publish_error counter
emqx_packets_publish_error 0
# TYPE emqx_packets_puback_received counter
emqx_packets_puback_received 0
# TYPE emqx_packets_puback_sent counter
emqx_packets_puback_sent 0
# TYPE emqx_packets_puback_missed counter
emqx_packets_puback_missed 0
# TYPE emqx_packets_pubrec_received counter
emqx_packets_pubrec_received 0
# TYPE emqx_packets_pubrec_sent counter
emqx_packets_pubrec_sent 0
# TYPE emqx_packets_pubrec_missed counter
emqx_packets_pubrec_missed 0
# TYPE emqx_packets_pubrel_received counter
emqx_packets_pubrel_received 0
# TYPE emqx_packets_pubrel_sent counter
emqx_packets_pubrel_sent 0
# TYPE emqx_packets_pubrel_missed counter
emqx_packets_pubrel_missed 0
# TYPE emqx_packets_pubcomp_received counter
emqx_packets_pubcomp_received 0
# TYPE emqx_packets_pubcomp_sent counter
emqx_packets_pubcomp_sent 0
# TYPE emqx_packets_pubcomp_missed counter
emqx_packets_pubcomp_missed 0
# TYPE emqx_packets_pingreq counter
emqx_packets_pingreq 0
# TYPE emqx_packets_pingresp counter
emqx_packets_pingresp 0
# TYPE emqx_bytes_received counter
emqx_bytes_received 0
# TYPE emqx_bytes_sent counter
emqx_bytes_sent 0
# TYPE emqx_connections_count gauge
emqx_connections_count 0
# TYPE emqx_connections_max gauge
emqx_connections_max 0
# TYPE emqx_retained_count gauge
emqx_retained_count 3
# TYPE emqx_retained_max gauge
emqx_retained_max 3
# TYPE emqx_routes_count gauge
emqx_routes_count 0
# TYPE emqx_routes_max gauge
emqx_routes_max 0
# TYPE emqx_sessions_count gauge
emqx_sessions_count 0
# TYPE emqx_sessions_max gauge
emqx_sessions_max 0
# TYPE emqx_subscriptions_count gauge
emqx_subscriptions_count 0
# TYPE emqx_subscriptions_max gauge
emqx_subscriptions_max 0
# TYPE emqx_topics_count gauge
emqx_topics_count 0
# TYPE emqx_topics_max gauge
emqx_topics_max 0
# TYPE emqx_vm_cpu_use gauge
emqx_vm_cpu_use 100.0
# TYPE emqx_vm_cpu_idle gauge
emqx_vm_cpu_idle 0.0
# TYPE emqx_vm_run_queue gauge
emqx_vm_run_queue 1
# TYPE emqx_vm_process_messages_in_queues gauge
emqx_vm_process_messages_in_queues 0
# TYPE emqx_messages_received counter
emqx_messages_received 0
# TYPE emqx_messages_sent counter
emqx_messages_sent 0
# TYPE emqx_messages_dropped counter
emqx_messages_dropped 0
# TYPE emqx_messages_retained counter
emqx_messages_retained 3
# TYPE emqx_messages_qos0_received counter
emqx_messages_qos0_received 0
# TYPE emqx_messages_qos0_sent counter
emqx_messages_qos0_sent 0
# TYPE emqx_messages_qos1_received counter
emqx_messages_qos1_received 0
# TYPE emqx_messages_qos1_sent counter
emqx_messages_qos1_sent 0
# TYPE emqx_messages_qos2_received counter
emqx_messages_qos2_received 0
# TYPE emqx_messages_qos2_expired counter
emqx_messages_qos2_expired 0
# TYPE emqx_messages_qos2_sent counter
emqx_messages_qos2_sent 0
# TYPE emqx_messages_qos2_dropped counter
emqx_messages_qos2_dropped 0
# TYPE emqx_messages_forward counter
emqx_messages_forward 0
```
License
-------

View File

@ -49,6 +49,18 @@ spec:
app.kubernetes.io/instance: {{ .Release.Name }}
spec:
volumes:
- name: emqx-loaded-plugins
configMap:
name: {{ include "emqx.fullname" . }}-loaded-plugins
items:
- key: loaded_plugins
path: loaded_plugins
- name: emqx-loaded-modules
configMap:
name: {{ include "emqx.fullname" . }}-loaded-modules
items:
- key: loaded_modules
path: loaded_modules
- name: emqx-acl
configMap:
name: {{ include "emqx.fullname" . }}-acl
@ -128,6 +140,12 @@ spec:
- name: emqx-acl
mountPath: "/opt/emqx/etc/acl.conf"
subPath: "acl.conf"
- name: emqx-loaded-plugins
mountPath: "/opt/emqx/data/loaded_plugins"
subPath: "loaded_plugins"
- name: emqx-loaded-modules
mountPath: "/opt/emqx/data/loaded_modules"
subPath: "loaded_modules"
{{ if .Values.emqxLicneseSecretName }}
- name: emqx-license
mountPath: "/opt/emqx/etc/emqx.lic"

View File

@ -11,4 +11,49 @@ metadata:
data:
{{- range $index, $value := .Values.emqxConfig}}
{{$index}}: "{{ $value }}"
{{- end}}
{{- end}}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-acl
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"acl.conf": |
{{ .Values.emqxAclConfig }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-plugins
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_plugins": |
{{ .Values.emqxLoadedPlugins }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-loaded-modules
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"loaded_modules": |
{{ .Values.emqxLoadedModules }}

View File

@ -1,13 +0,0 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "emqx.fullname" . }}-acl
namespace: {{ .Release.Namespace }}
labels:
app.kubernetes.io/name: {{ include "emqx.name" . }}
helm.sh/chart: {{ include "emqx.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
data:
"acl.conf": |
{{ .Values.emqxAclConfig }}

View File

@ -74,6 +74,23 @@ emqxAclConfig: >
{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
{allow, all}.
emqxLoadedPlugins: >
{emqx_management, true}.
{emqx_recon, true}.
{emqx_retainer, true}.
{emqx_dashboard, true}.
{emqx_telemetry, true}.
{emqx_rule_engine, true}.
{emqx_bridge_mqtt, false}.
emqxLoadedModules: >
{emqx_mod_acl_internal, true}.
{emqx_mod_presence, true}.
{emqx_mod_delayed, false}.
{emqx_mod_rewrite, false}.
{emqx_mod_subscription, false}.
{emqx_mod_topic_metrics, false}.
## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicneseSecretName"
## Example:
## kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic

View File

@ -274,7 +274,8 @@ handle_in(?CONNECT_PACKET(ConnPkt), Channel) ->
handle_out(connack, ReasonCode, NChannel)
end;
handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties), Channel = #channel{conn_state = ConnState}) ->
handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties),
Channel = #channel{conn_state = ConnState}) ->
case enhanced_auth(Packet, Channel) of
{ok, NProperties, NChannel} ->
case ConnState of
@ -291,7 +292,8 @@ handle_in(Packet = ?AUTH_PACKET(?RC_CONTINUE_AUTHENTICATION, _Properties), Chann
handle_out(connack, NReasonCode, NChannel)
end;
handle_in(Packet = ?AUTH_PACKET(?RC_RE_AUTHENTICATE, _Properties), Channel = #channel{conn_state = connected}) ->
handle_in(Packet = ?AUTH_PACKET(?RC_RE_AUTHENTICATE, _Properties),
Channel = #channel{conn_state = connected}) ->
case enhanced_auth(Packet, Channel) of
{ok, NProperties, NChannel} ->
handle_out(auth, {?RC_SUCCESS, NProperties}, NChannel);
@ -412,8 +414,10 @@ handle_in(Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
handle_in(?PACKET(?PINGREQ), Channel) ->
{ok, ?PACKET(?PINGRESP), Channel};
handle_in(?DISCONNECT_PACKET(ReasonCode, Properties), Channel = #channel{conninfo = ConnInfo}) ->
NChannel = maybe_clean_will_msg(ReasonCode, Channel#channel{conninfo = ConnInfo#{disconn_props => Properties}}),
handle_in(?DISCONNECT_PACKET(ReasonCode, Properties),
Channel = #channel{conninfo = ConnInfo}) ->
NConnInfo = ConnInfo#{disconn_props => Properties},
NChannel = maybe_clean_will_msg(ReasonCode, Channel#channel{conninfo = NConnInfo}),
process_disconnect(ReasonCode, Properties, NChannel);
handle_in(?AUTH_PACKET(), Channel) ->
@ -444,7 +448,9 @@ handle_in(Packet, Channel) ->
%% Process Connect
%%--------------------------------------------------------------------
process_connect(AckProps, Channel = #channel{conninfo = #{clean_start := CleanStart} = ConnInfo, clientinfo = ClientInfo}) ->
process_connect(AckProps, Channel = #channel{conninfo = ConnInfo,
clientinfo = ClientInfo}) ->
#{clean_start := CleanStart} = ConnInfo,
case emqx_cm:open_session(CleanStart, ClientInfo, ConnInfo) of
{ok, #{session := Session, present := false}} ->
NChannel = Channel#channel{session = Session},
@ -598,7 +604,8 @@ process_subscribe([{TopicFilter, SubOpts}|More], SubProps, Channel, Acc) ->
{RC, NChannel} = do_subscribe(TopicFilter, SubOpts#{sub_props => SubProps}, Channel),
process_subscribe(More, SubProps, NChannel, [RC|Acc]);
{error, RC} ->
?LOG(warning, "Cannot subscribe ~s due to ~s.", [TopicFilter, emqx_reason_codes:text(RC)]),
?LOG(warning, "Cannot subscribe ~s due to ~s.",
[TopicFilter, emqx_reason_codes:text(RC)]),
process_subscribe(More, SubProps, Channel, [RC|Acc])
end.
@ -611,7 +618,8 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel =
{ok, NSession} ->
{QoS, Channel#channel{session = NSession}};
{error, RC} ->
?LOG(warning, "Cannot subscribe ~s due to ~s.", [TopicFilter, emqx_reason_codes:text(RC)]),
?LOG(warning, "Cannot subscribe ~s due to ~s.",
[TopicFilter, emqx_reason_codes:text(RC)]),
{RC, Channel}
end.
@ -729,7 +737,10 @@ handle_out(connack, {?RC_SUCCESS, SP, Props}, Channel = #channel{conninfo = Conn
fun enrich_response_information/2,
fun enrich_assigned_clientid/2
], Props, Channel),
NAckProps = run_hooks('client.connack', [ConnInfo, emqx_reason_codes:name(?RC_SUCCESS)], AckProps),
NAckProps = run_hooks('client.connack',
[ConnInfo, emqx_reason_codes:name(?RC_SUCCESS)],
AckProps
),
return_connack(?CONNACK_PACKET(?RC_SUCCESS, SP, NAckProps),
ensure_keepalive(NAckProps, Channel));
@ -764,17 +775,17 @@ handle_out(pubcomp, {PacketId, ReasonCode}, Channel) ->
{ok, ?PUBCOMP_PACKET(PacketId, ReasonCode), Channel};
handle_out(suback, {PacketId, ReasonCodes}, Channel = ?IS_MQTT_V5) ->
return_suback(?SUBACK_PACKET(PacketId, ReasonCodes), Channel);
return_sub_unsub_ack(?SUBACK_PACKET(PacketId, ReasonCodes), Channel);
handle_out(suback, {PacketId, ReasonCodes}, Channel) ->
ReasonCodes1 = [emqx_reason_codes:compat(suback, RC) || RC <- ReasonCodes],
return_suback(?SUBACK_PACKET(PacketId, ReasonCodes1), Channel);
return_sub_unsub_ack(?SUBACK_PACKET(PacketId, ReasonCodes1), Channel);
handle_out(unsuback, {PacketId, ReasonCodes}, Channel = ?IS_MQTT_V5) ->
return_unsuback(?UNSUBACK_PACKET(PacketId, ReasonCodes), Channel);
return_sub_unsub_ack(?UNSUBACK_PACKET(PacketId, ReasonCodes), Channel);
handle_out(unsuback, {PacketId, _ReasonCodes}, Channel) ->
return_unsuback(?UNSUBACK_PACKET(PacketId), Channel);
return_sub_unsub_ack(?UNSUBACK_PACKET(PacketId), Channel);
handle_out(disconnect, ReasonCode, Channel) when is_integer(ReasonCode) ->
ReasonName = disconnect_reason(ReasonCode),
@ -847,10 +858,7 @@ do_deliver(Publishes, Channel) when is_list(Publishes) ->
%% Handle out suback
%%--------------------------------------------------------------------
return_suback(Packet, Channel) ->
{ok, [{outgoing, Packet}, {event, updated}], Channel}.
return_unsuback(Packet, Channel) ->
return_sub_unsub_ack(Packet, Channel) ->
{ok, [{outgoing, Packet}, {event, updated}], Channel}.
%%--------------------------------------------------------------------
@ -1187,11 +1195,10 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
%%--------------------------------------------------------------------
%% Auth Connect
auth_connect(#mqtt_packet_connect{clientid = ClientId,
username = Username,
password = Password
},
auth_connect(#mqtt_packet_connect{password = Password},
#channel{clientinfo = ClientInfo} = Channel) ->
#{clientid := ClientId,
username := Username} = ClientInfo,
case emqx_access_control:authenticate(ClientInfo#{password => Password}) of
{ok, AuthResult} ->
is_anonymous(AuthResult) andalso
@ -1224,7 +1231,10 @@ enhanced_auth(?CONNECT_PACKET(#mqtt_packet_connect{
end;
enhanced_auth(?AUTH_PACKET(_ReasonCode, Properties), Channel = #channel{conninfo = ConnInfo}) ->
AuthMethod = emqx_mqtt_props:get('Authentication-Method', emqx_mqtt_props:get(conn_props, ConnInfo, #{}), undefined),
AuthMethod = emqx_mqtt_props:get('Authentication-Method',
emqx_mqtt_props:get(conn_props, ConnInfo, #{}),
undefined
),
NAuthMethod = emqx_mqtt_props:get('Authentication-Method', Properties, undefined),
AuthData = emqx_mqtt_props:get('Authentication-Data', Properties, undefined),
case NAuthMethod =:= undefined orelse NAuthMethod =/= AuthMethod of
@ -1241,12 +1251,14 @@ do_enhanced_auth(undefined, _AuthData, Channel) ->
do_enhanced_auth(_AuthMethod, undefined, Channel) ->
{error, emqx_reason_codes:connack_error(not_authorized), Channel};
do_enhanced_auth(AuthMethod, AuthData, Channel = #channel{auth_cache = Cache}) ->
case run_hooks('client.enhanced_authenticate',[AuthMethod, AuthData], Cache) of
case run_hooks('client.enhanced_authenticate', [AuthMethod, AuthData], Cache) of
{ok, NAuthData, NCache} ->
NProperties = #{'Authentication-Method' => AuthMethod, 'Authentication-Data' => NAuthData},
NProperties = #{'Authentication-Method' => AuthMethod,
'Authentication-Data' => NAuthData},
{ok, NProperties, Channel#channel{auth_cache = NCache}};
{continue, NAuthData, NCache} ->
NProperties = #{'Authentication-Method' => AuthMethod, 'Authentication-Data' => NAuthData},
NProperties = #{'Authentication-Method' => AuthMethod,
'Authentication-Data' => NAuthData},
{continue, NProperties, Channel#channel{auth_cache = NCache}};
_ ->
{error, emqx_reason_codes:connack_error(not_authorized), Channel}
@ -1288,9 +1300,10 @@ packing_alias(Packet = #mqtt_packet{
properties = Prop
} = Publish
},
Channel = ?IS_MQTT_V5 = #channel{topic_aliases = TopicAliases, alias_maximum = Limits}) ->
Channel = ?IS_MQTT_V5 = #channel{topic_aliases = TopicAliases,
alias_maximum = Limits}) ->
case find_alias(outbound, Topic, TopicAliases) of
{ok, AliasId} ->
{ok, AliasId} ->
NPublish = Publish#mqtt_packet_publish{
topic_name = <<>>,
properties = maps:merge(Prop, #{'Topic-Alias' => AliasId})
@ -1592,7 +1605,7 @@ run_hooks(Name, Args, Acc) ->
-compile({inline, [find_alias/3, save_alias/4]}).
find_alias(_, _ ,undefined) -> false;
find_alias(_, _, undefined) -> false;
find_alias(inbound, AliasId, _TopicAliases = #{inbound := Aliases}) ->
maps:find(AliasId, Aliases);
find_alias(outbound, Topic, _TopicAliases = #{outbound := Aliases}) ->

View File

@ -40,13 +40,6 @@
-export([ get_ets_list/0
, get_ets_info/0
, get_ets_info/1
, get_ets_object/0
, get_ets_object/1
]).
-export([ get_port_types/0
, get_port_info/0
, get_port_info/1
]).
-export([cpu_util/0]).
@ -187,7 +180,7 @@ get_system_info(Key) ->
format_system_info(allocated_areas, List) ->
[convert_allocated_areas(Value) || Value <- List];
format_system_info(allocator, {_,_,_,List}) ->
format_system_info(allocator, {_, _, _, List}) ->
List;
format_system_info(dist_ctrl, List) ->
lists:map(fun({Node, Socket}) ->
@ -237,7 +230,7 @@ scheduler_usage(Interval) when is_integer(Interval) ->
scheduler_usage_diff(First, Last).
scheduler_usage_diff(First, Last) ->
lists:map(fun({{I, A0, T0},{I, A1, T1}}) ->
lists:map(fun({{I, A0, T0}, {I, A1, T1}}) ->
{I, (A1 - A0)/(T1 - T0)}
end, lists:zip(lists:sort(First), lists:sort(Last))).
@ -287,7 +280,7 @@ alloc(Type) ->
allocators() ->
UtilAllocators = erlang:system_info(alloc_util_allocators),
Allocators = [sys_alloc, mseg_alloc|UtilAllocators],
[{{A, N},lists:sort(proplists:delete(versions, Props))} ||
[{{A, N}, lists:sort(proplists:delete(versions, Props))} ||
A <- Allocators, Allocs <- [erlang:system_info({allocator, A})],
Allocs =/= false, {_, N, Props} <- Allocs].
@ -348,111 +341,6 @@ get_ets_info(Tab) ->
mapping(Entries)
end.
get_ets_object() ->
[{Tab, get_ets_object(Tab)} || Tab <- ets:all()].
get_ets_object(Tab) ->
TabInfo = ets:info(Tab),
Size = proplists:get_value(size, TabInfo),
NameTab = proplists:get_value(named_table, TabInfo),
if (Size == 0) or (NameTab == false) ->
[];
true ->
ets:tab2list(Tab)
end.
get_port_types() ->
lists:usort(fun({KA, VA},{KB, VB})-> {VA, KB} >{VB, KA} end,
ports_type_count([Type || {_Port, Type} <- ports_type_list()])).
get_port_info() ->
[get_port_info(Port) ||Port <- erlang:ports()].
get_port_info(PortTerm) ->
Port = transform_port(PortTerm),
[port_info(Port, Type) || Type <- [meta, signals, io, memory_used, specific]].
port_info(Port, meta) ->
{meta, List} = port_info_type(Port, meta, [id, name, os_pid]),
case port_info(Port, registered_name) of
[] -> {meta, List};
Name -> {meta, [Name | List]}
end;
port_info(PortTerm, signals) ->
port_info_type(PortTerm, signals, [connected, links, monitors]);
port_info(PortTerm, io) ->
port_info_type(PortTerm, io, [input, output]);
port_info(PortTerm, memory_used) ->
port_info_type(PortTerm, memory_used, [memory, queue_size]);
port_info(PortTerm, specific) ->
Port = transform_port(PortTerm),
Props = case erlang:port_info(Port, name) of
{_, Type} when Type =:= "udp_inet";
Type =:= "tcp_inet";
Type =:= "sctp_inet" ->
try inet:getstat(Port) of
{ok, Stats} -> [{statistics, Stats}];
{error, _} -> []
catch
_Error:_Reason -> []
end ++
try inet:peername(Port) of
{ok, Peer} -> [{peername, Peer}];
_ -> []
catch
_Error:_Reason -> []
end ++
try inet:sockname(Port) of
{ok, Local} -> [{sockname, Local}];
{error, _} -> []
catch
_Error:_Reason -> []
end ++
try inet:getopts(Port, ?SOCKET_OPTS ) of
{ok, Opts} -> [{options, Opts}];
{error, _} -> []
catch
_Error:_Reason -> []
end;
{_, "efile"} ->
[];
_ ->
[]
end,
{specific, Props};
port_info(PortTerm, Key) when is_atom(Key) ->
Port = transform_port(PortTerm),
erlang:port_info(Port, Key).
port_info_type(PortTerm, Type, Keys) ->
Port = transform_port(PortTerm),
{Type, [erlang:port_info(Port, Key) || Key <- Keys]}.
transform_port(Port) when is_port(Port) -> Port;
transform_port("#Port<0." ++ Id) ->
N = list_to_integer(lists:sublist(Id, length(Id) - 1)),
transform_port(N);
transform_port(N) when is_integer(N) ->
Name = iolist_to_binary(atom_to_list(node())),
NameLen = iolist_size(Name),
Vsn = binary:last(term_to_binary(self())),
Bin = <<131, 102, 100, NameLen:2/unit:8, Name:NameLen/binary, N:4/unit:8, Vsn:8>>,
binary_to_term(Bin).
ports_type_list() ->
[{Port, PortType} || Port <- erlang:ports(),
{_, PortType} <- [erlang:port_info(Port, name)]].
ports_type_count(Types) ->
DictTypes = lists:foldl(fun(Type, Acc)->
dict:update_counter(Type, 1, Acc)
end, dict:new(), Types),
dict:to_list(DictTypes).
mapping(Entries) ->
mapping(Entries, []).
mapping([], Acc) -> Acc;

100
sync-apps.sh Executable file
View File

@ -0,0 +1,100 @@
#!/bin/bash
set -euo pipefail
force="${1:-no}"
apps=(
# "emqx_auth_http" # permanently diverged
# "emqx_web_hook" # permanently diverged
"emqx_auth_jwt"
"emqx_auth_ldap"
"emqx_auth_mongo"
"emqx_auth_mysql"
"emqx_auth_pgsql"
"emqx_auth_redis"
"emqx_bridge_mqtt"
"emqx_coap"
"emqx_dashboard"
"emqx_exhook"
"emqx_exproto"
"emqx_lua_hook"
"emqx_lwm2m"
"emqx_management"
"emqx_plugin_template"
"emqx_prometheus"
"emqx_psk_file"
"emqx_recon"
"emqx_retainer"
"emqx_rule_engine"
"emqx_sasl"
"emqx_sn"
"emqx_stomp"
"emqx_telemetry"
)
if git status --porcelain | grep -qE 'apps/'; then
echo 'apps dir is not git-clear, refuse to sync'
# exit 1
fi
mkdir -p tmp/
download_zip() {
local app="$1"
local ref="$2"
local vsn="$(echo "$ref" | tr '/' '-')"
local file="tmp/${app}-${vsn}.zip"
if [ -f "$file" ] && [ "$force" != "force" ]; then
return 0
fi
local repo="$(echo "$app" | sed 's#_#-#g')"
local url="https://github.com/emqx/$repo/archive/$ref.zip"
echo "downloading ${url}"
curl -fLsS -o "$file" "$url"
}
default_vsn="dev/v4.3.0"
download_zip "emqx_auth_mnesia" "e4.2.3"
for app in ${apps[@]}; do
download_zip "$app" "$default_vsn"
done
extract_zip(){
local app="$1"
local ref="$2"
local vsn_arg="${3:-}"
local vsn_dft="$(echo "$ref" | tr '/' '-')"
local vsn
if [ -n "$vsn_arg" ]; then
vsn="$vsn_arg"
else
vsn="$vsn_dft"
fi
local file="tmp/${app}-${vsn_dft}.zip"
local repo="$(echo "$app" | sed 's#_#-#g')"
rm -rf "apps/${app}/"
unzip "$file" -d apps/
mv "apps/${repo}-${vsn}/" "apps/$app/"
}
extract_zip "emqx_auth_mnesia" "e4.2.3" "e4.2.3"
for app in ${apps[@]}; do
extract_zip "$app" "$default_vsn"
done
cleanup_app(){
local app="$1"
pushd "apps/$app"
rm -f Makefile rebar.config.script LICENSE src/*.app.src.script src/*.appup.src
rm -rf ".github" ".ci"
# restore rebar.config and app.src
git checkout rebar.config
git checkout src/*.app.src
popd
}
apps+=( "emqx_auth_mnesia" )
for app in ${apps[@]}; do
cleanup_app $app
done

View File

@ -64,30 +64,6 @@ t_get_ets_info(_Config) ->
end
end, false, EtsInfos)).
t_get_ets_object(_Config) ->
ets:new(test, [named_table]),
[] = emqx_vm:get_ets_object(test),
ets:insert(test, {k, v}),
[{k, v}] = emqx_vm:get_ets_object(test).
t_get_port_types(_Config) ->
emqx_vm:get_port_types().
t_get_port_info(_Config) ->
emqx_vm:get_port_info(),
spawn(fun easy_server/0),
ct:sleep(100),
{ok, Sock} = gen_tcp:connect("localhost", 5678, [binary, {packet, 0}]),
emqx_vm:get_port_info(),
ok = gen_tcp:close(Sock),
[_Port | _] = erlang:ports().
t_transform_port(_Config) ->
[Port | _] = erlang:ports(),
?assertEqual(Port, emqx_vm:transform_port(Port)),
<<131, 102, 100, NameLen:2/unit:8, _Name:NameLen/binary, N:4/unit:8, _Vsn:8>> = erlang:term_to_binary(Port),
?assertEqual(Port, emqx_vm:transform_port("#Port<0." ++ integer_to_list(N) ++ ">")).
t_scheduler_usage(_Config) ->
emqx_vm:scheduler_usage(5000).